2023-10-09T03:48:58.687 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T03:48:58.687 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --verbose --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/orig.config.yaml 2023-10-09T03:48:58.752 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T03:48:58.852 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7417455' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 kernel: hwe: true workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi082.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi153.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T03:48:58.852 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T03:48:58.853 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T03:48:58.853 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T03:48:58.855 INFO:teuthology.task.internal:Checking packages... 2023-10-09T03:48:58.882 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T03:48:58.882 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:48:58.883 INFO:teuthology.packaging:ref: None 2023-10-09T03:48:58.883 INFO:teuthology.packaging:tag: None 2023-10-09T03:48:58.883 INFO:teuthology.packaging:branch: reef 2023-10-09T03:48:58.883 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:48:58.883 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-09T03:48:59.119 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610-g397de1b0-1focal 2023-10-09T03:48:59.119 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T03:48:59.126 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T03:48:59.126 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T03:48:59.140 INFO:teuthology.task.internal:Saving configuration 2023-10-09T03:48:59.162 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T03:48:59.168 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T03:48:59.195 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi082.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:42:45.072359', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T03:48:59.221 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi153.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:42:45.070855', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T03:48:59.221 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T03:48:59.227 INFO:teuthology.task.internal:roles: ubuntu@smithi082.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T03:48:59.228 INFO:teuthology.task.internal:roles: ubuntu@smithi153.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T03:48:59.228 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T03:48:59.300 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4ffd2dd160>, signals=[15]) 2023-10-09T03:48:59.301 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T03:48:59.307 INFO:teuthology.task.internal:Opening connections... 2023-10-09T03:48:59.307 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi082.front.sepia.ceph.com 2023-10-09T03:48:59.309 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:48:59.358 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi153.front.sepia.ceph.com 2023-10-09T03:48:59.359 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:48:59.407 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T03:48:59.413 DEBUG:teuthology.orchestra.run.smithi082:> uname -m 2023-10-09T03:48:59.435 INFO:teuthology.orchestra.run.smithi082.stdout:x86_64 2023-10-09T03:48:59.436 DEBUG:teuthology.orchestra.run.smithi082:> cat /etc/os-release 2023-10-09T03:48:59.489 INFO:teuthology.orchestra.run.smithi082.stdout:NAME="Ubuntu" 2023-10-09T03:48:59.489 INFO:teuthology.orchestra.run.smithi082.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T03:48:59.489 INFO:teuthology.orchestra.run.smithi082.stdout:ID=ubuntu 2023-10-09T03:48:59.489 INFO:teuthology.orchestra.run.smithi082.stdout:ID_LIKE=debian 2023-10-09T03:48:59.489 INFO:teuthology.orchestra.run.smithi082.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:VERSION_ID="20.04" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:VERSION_CODENAME=focal 2023-10-09T03:48:59.490 INFO:teuthology.orchestra.run.smithi082.stdout:UBUNTU_CODENAME=focal 2023-10-09T03:48:59.491 INFO:teuthology.lock.ops:Updating smithi082.front.sepia.ceph.com on lock server 2023-10-09T03:48:59.517 DEBUG:teuthology.orchestra.run.smithi153:> uname -m 2023-10-09T03:48:59.523 INFO:teuthology.orchestra.run.smithi153.stdout:x86_64 2023-10-09T03:48:59.524 DEBUG:teuthology.orchestra.run.smithi153:> cat /etc/os-release 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:NAME="Ubuntu" 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:ID=ubuntu 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:ID_LIKE=debian 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION_ID="20.04" 2023-10-09T03:48:59.530 INFO:teuthology.orchestra.run.smithi153.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T03:48:59.531 INFO:teuthology.orchestra.run.smithi153.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T03:48:59.531 INFO:teuthology.orchestra.run.smithi153.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T03:48:59.531 INFO:teuthology.orchestra.run.smithi153.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T03:48:59.531 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION_CODENAME=focal 2023-10-09T03:48:59.531 INFO:teuthology.orchestra.run.smithi153.stdout:UBUNTU_CODENAME=focal 2023-10-09T03:48:59.531 INFO:teuthology.lock.ops:Updating smithi153.front.sepia.ceph.com on lock server 2023-10-09T03:48:59.550 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T03:48:59.559 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T03:48:59.564 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T03:48:59.564 DEBUG:teuthology.orchestra.run.smithi082:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:48:59.566 DEBUG:teuthology.orchestra.run.smithi153:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:48:59.571 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T03:48:59.578 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T03:48:59.578 DEBUG:teuthology.orchestra.run.smithi082:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:48:59.580 DEBUG:teuthology.orchestra.run.smithi153:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:48:59.631 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T03:48:59.685 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T03:48:59.700 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T03:48:59.700 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-09T03:48:59.700 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-09T03:48:59.700 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-09T03:48:59.700 DEBUG:teuthology.orchestra.run.smithi082:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:48:59.701 DEBUG:teuthology.orchestra.run.smithi153:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:48:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:48:59.707 DEBUG:teuthology.orchestra.run.smithi082:> uname -r 2023-10-09T03:48:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:48:59.708 DEBUG:teuthology.orchestra.run.smithi153:> uname -r 2023-10-09T03:48:59.715 INFO:teuthology.orchestra.run.smithi082.stdout:5.4.0-124-generic 2023-10-09T03:48:59.715 INFO:teuthology.task.kernel:Running kernel on smithi082: 5.4.0-124-generic 2023-10-09T03:48:59.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-get clean 2023-10-09T03:48:59.716 INFO:teuthology.orchestra.run.smithi153.stdout:5.4.0-124-generic 2023-10-09T03:48:59.716 INFO:teuthology.task.kernel:Running kernel on smithi153: 5.4.0-124-generic 2023-10-09T03:48:59.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-get clean 2023-10-09T03:48:59.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-get update 2023-10-09T03:48:59.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-get update 2023-10-09T03:49:00.026 INFO:teuthology.orchestra.run.smithi082.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T03:49:00.156 INFO:teuthology.orchestra.run.smithi153.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T03:49:00.159 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:49:00.174 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:49:00.241 INFO:teuthology.orchestra.run.smithi153.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T03:49:00.242 INFO:teuthology.orchestra.run.smithi082.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T03:49:00.304 INFO:teuthology.orchestra.run.smithi082.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T03:49:00.395 INFO:teuthology.orchestra.run.smithi082.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T03:49:00.435 INFO:teuthology.orchestra.run.smithi082.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T03:49:00.437 INFO:teuthology.orchestra.run.smithi082.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T03:49:00.437 INFO:teuthology.orchestra.run.smithi082.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T03:49:00.452 INFO:teuthology.orchestra.run.smithi082.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T03:49:00.452 INFO:teuthology.orchestra.run.smithi082.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T03:49:00.455 INFO:teuthology.orchestra.run.smithi082.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T03:49:00.455 INFO:teuthology.orchestra.run.smithi082.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T03:49:00.460 INFO:teuthology.orchestra.run.smithi082.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T03:49:00.464 INFO:teuthology.orchestra.run.smithi082.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T03:49:00.465 INFO:teuthology.orchestra.run.smithi082.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T03:49:00.471 INFO:teuthology.orchestra.run.smithi082.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T03:49:00.471 INFO:teuthology.orchestra.run.smithi082.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T03:49:00.472 INFO:teuthology.orchestra.run.smithi082.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T03:49:00.472 INFO:teuthology.orchestra.run.smithi082.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T03:49:00.585 INFO:teuthology.orchestra.run.smithi082.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T03:49:00.585 INFO:teuthology.orchestra.run.smithi153.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T03:49:00.651 INFO:teuthology.orchestra.run.smithi153.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T03:49:00.727 INFO:teuthology.orchestra.run.smithi082.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T03:49:00.791 INFO:teuthology.orchestra.run.smithi153.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T03:49:00.974 INFO:teuthology.orchestra.run.smithi082.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T03:49:00.997 INFO:teuthology.orchestra.run.smithi153.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T03:49:01.123 INFO:teuthology.orchestra.run.smithi153.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T03:49:01.138 INFO:teuthology.orchestra.run.smithi082.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T03:49:01.151 INFO:teuthology.orchestra.run.smithi082.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T03:49:01.152 INFO:teuthology.orchestra.run.smithi082.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T03:49:01.152 INFO:teuthology.orchestra.run.smithi082.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T03:49:01.185 INFO:teuthology.orchestra.run.smithi153.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T03:49:01.198 INFO:teuthology.orchestra.run.smithi153.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T03:49:01.199 INFO:teuthology.orchestra.run.smithi153.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T03:49:01.213 INFO:teuthology.orchestra.run.smithi082.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T03:49:01.233 INFO:teuthology.orchestra.run.smithi082.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T03:49:01.267 INFO:teuthology.orchestra.run.smithi153.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T03:49:01.268 INFO:teuthology.orchestra.run.smithi153.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T03:49:01.274 INFO:teuthology.orchestra.run.smithi153.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T03:49:01.274 INFO:teuthology.orchestra.run.smithi153.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T03:49:01.296 INFO:teuthology.orchestra.run.smithi153.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T03:49:01.334 INFO:teuthology.orchestra.run.smithi153.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T03:49:01.347 INFO:teuthology.orchestra.run.smithi153.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T03:49:01.347 INFO:teuthology.orchestra.run.smithi153.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T03:49:01.348 INFO:teuthology.orchestra.run.smithi153.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T03:49:01.397 INFO:teuthology.orchestra.run.smithi082.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T03:49:01.425 INFO:teuthology.orchestra.run.smithi153.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T03:49:01.432 INFO:teuthology.orchestra.run.smithi153.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T03:49:01.485 INFO:teuthology.orchestra.run.smithi153.stdout:Get:23 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T03:49:01.596 INFO:teuthology.orchestra.run.smithi153.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T03:49:01.902 INFO:teuthology.orchestra.run.smithi153.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T03:49:01.908 INFO:teuthology.orchestra.run.smithi153.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T03:49:01.910 INFO:teuthology.orchestra.run.smithi153.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T03:49:01.918 INFO:teuthology.orchestra.run.smithi153.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T03:49:01.920 INFO:teuthology.orchestra.run.smithi153.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T03:49:01.984 INFO:teuthology.orchestra.run.smithi082.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T03:49:02.094 INFO:teuthology.orchestra.run.smithi082.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T03:49:02.114 INFO:teuthology.orchestra.run.smithi082.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T03:49:02.116 INFO:teuthology.orchestra.run.smithi082.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T03:49:02.117 INFO:teuthology.orchestra.run.smithi082.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T03:49:02.118 INFO:teuthology.orchestra.run.smithi082.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T03:49:02.119 INFO:teuthology.orchestra.run.smithi082.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T03:49:02.119 INFO:teuthology.orchestra.run.smithi082.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T03:49:02.122 INFO:teuthology.orchestra.run.smithi082.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T03:49:02.124 INFO:teuthology.orchestra.run.smithi082.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T03:49:02.236 INFO:teuthology.orchestra.run.smithi153.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T03:49:02.276 INFO:teuthology.orchestra.run.smithi153.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T03:49:02.288 INFO:teuthology.orchestra.run.smithi082.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T03:49:02.290 INFO:teuthology.orchestra.run.smithi153.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T03:49:02.290 INFO:teuthology.orchestra.run.smithi153.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T03:49:02.291 INFO:teuthology.orchestra.run.smithi153.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T03:49:02.293 INFO:teuthology.orchestra.run.smithi153.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T03:49:02.293 INFO:teuthology.orchestra.run.smithi153.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T03:49:02.293 INFO:teuthology.orchestra.run.smithi153.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T03:49:02.295 INFO:teuthology.orchestra.run.smithi153.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T03:49:02.320 INFO:teuthology.orchestra.run.smithi153.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T03:49:02.471 INFO:teuthology.orchestra.run.smithi082.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T03:49:02.484 INFO:teuthology.orchestra.run.smithi153.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T03:49:02.494 INFO:teuthology.orchestra.run.smithi082.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T03:49:02.668 INFO:teuthology.orchestra.run.smithi153.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T03:49:02.690 INFO:teuthology.orchestra.run.smithi153.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T03:49:04.492 INFO:teuthology.orchestra.run.smithi082.stdout:Fetched 17.6 MB in 3s (6,865 kB/s) 2023-10-09T03:49:05.081 INFO:teuthology.orchestra.run.smithi153.stdout:Fetched 17.6 MB in 3s (5,740 kB/s) 2023-10-09T03:49:05.833 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:49:05.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T03:49:05.929 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:49:06.130 INFO:teuthology.orchestra.run.smithi082.stdout:Building dependency tree... 2023-10-09T03:49:06.131 INFO:teuthology.orchestra.run.smithi082.stdout:Reading state information... 2023-10-09T03:49:06.309 INFO:teuthology.orchestra.run.smithi082.stdout:The following additional packages will be installed: 2023-10-09T03:49:06.309 INFO:teuthology.orchestra.run.smithi082.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T03:49:06.309 INFO:teuthology.orchestra.run.smithi082.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T03:49:06.309 INFO:teuthology.orchestra.run.smithi082.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T03:49:06.310 INFO:teuthology.orchestra.run.smithi082.stdout:Suggested packages: 2023-10-09T03:49:06.310 INFO:teuthology.orchestra.run.smithi082.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T03:49:06.310 INFO:teuthology.orchestra.run.smithi082.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T03:49:06.362 INFO:teuthology.orchestra.run.smithi082.stdout:The following NEW packages will be installed: 2023-10-09T03:49:06.362 INFO:teuthology.orchestra.run.smithi082.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T03:49:06.363 INFO:teuthology.orchestra.run.smithi082.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T03:49:06.363 INFO:teuthology.orchestra.run.smithi082.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T03:49:06.363 INFO:teuthology.orchestra.run.smithi082.stdout: thermald upower usbmuxd 2023-10-09T03:49:06.415 INFO:teuthology.orchestra.run.smithi082.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T03:49:06.415 INFO:teuthology.orchestra.run.smithi082.stdout:Need to get 90.9 MB of archives. 2023-10-09T03:49:06.415 INFO:teuthology.orchestra.run.smithi082.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T03:49:06.416 INFO:teuthology.orchestra.run.smithi082.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T03:49:06.452 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:49:06.471 INFO:teuthology.orchestra.run.smithi082.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T03:49:06.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T03:49:06.478 INFO:teuthology.orchestra.run.smithi082.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T03:49:06.481 INFO:teuthology.orchestra.run.smithi082.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T03:49:06.496 INFO:teuthology.orchestra.run.smithi082.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T03:49:06.503 INFO:teuthology.orchestra.run.smithi082.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T03:49:06.559 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:49:06.760 INFO:teuthology.orchestra.run.smithi153.stdout:Building dependency tree... 2023-10-09T03:49:06.761 INFO:teuthology.orchestra.run.smithi153.stdout:Reading state information... 2023-10-09T03:49:06.787 INFO:teuthology.orchestra.run.smithi082.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T03:49:06.890 INFO:teuthology.orchestra.run.smithi082.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T03:49:06.937 INFO:teuthology.orchestra.run.smithi153.stdout:The following additional packages will be installed: 2023-10-09T03:49:06.937 INFO:teuthology.orchestra.run.smithi153.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T03:49:06.938 INFO:teuthology.orchestra.run.smithi153.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T03:49:06.938 INFO:teuthology.orchestra.run.smithi153.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T03:49:06.939 INFO:teuthology.orchestra.run.smithi153.stdout:Suggested packages: 2023-10-09T03:49:06.939 INFO:teuthology.orchestra.run.smithi153.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T03:49:06.939 INFO:teuthology.orchestra.run.smithi153.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T03:49:06.992 INFO:teuthology.orchestra.run.smithi153.stdout:The following NEW packages will be installed: 2023-10-09T03:49:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T03:49:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T03:49:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T03:49:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: thermald upower usbmuxd 2023-10-09T03:49:07.050 INFO:teuthology.orchestra.run.smithi153.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T03:49:07.051 INFO:teuthology.orchestra.run.smithi153.stdout:Need to get 90.9 MB of archives. 2023-10-09T03:49:07.051 INFO:teuthology.orchestra.run.smithi153.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T03:49:07.051 INFO:teuthology.orchestra.run.smithi153.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T03:49:07.117 INFO:teuthology.orchestra.run.smithi153.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T03:49:07.123 INFO:teuthology.orchestra.run.smithi153.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T03:49:07.128 INFO:teuthology.orchestra.run.smithi153.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T03:49:07.145 INFO:teuthology.orchestra.run.smithi153.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T03:49:07.154 INFO:teuthology.orchestra.run.smithi153.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T03:49:07.484 INFO:teuthology.orchestra.run.smithi153.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T03:49:07.590 INFO:teuthology.orchestra.run.smithi082.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T03:49:07.590 INFO:teuthology.orchestra.run.smithi082.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T03:49:07.591 INFO:teuthology.orchestra.run.smithi082.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T03:49:07.597 INFO:teuthology.orchestra.run.smithi082.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T03:49:07.599 INFO:teuthology.orchestra.run.smithi082.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T03:49:07.603 INFO:teuthology.orchestra.run.smithi153.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T03:49:08.251 INFO:teuthology.orchestra.run.smithi082.stdout:Fetched 90.9 MB in 1s (74.0 MB/s) 2023-10-09T03:49:08.457 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T03:49:08.501 INFO:teuthology.orchestra.run.smithi153.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T03:49:08.501 INFO:teuthology.orchestra.run.smithi153.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T03:49:08.502 INFO:teuthology.orchestra.run.smithi153.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T03:49:08.508 INFO:teuthology.orchestra.run.smithi153.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T03:49:08.510 INFO:teuthology.orchestra.run.smithi153.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T03:49:09.222 INFO:teuthology.orchestra.run.smithi153.stdout:Fetched 90.9 MB in 2s (60.0 MB/s) 2023-10-09T03:49:09.448 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T03:49:10.183 INFO:teuthology.orchestra.run.smithi082.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T03:49:10.187 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T03:49:10.295 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T03:49:10.648 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T03:49:10.661 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T03:49:10.698 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T03:49:11.043 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T03:49:11.056 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T03:49:11.101 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T03:49:11.177 INFO:teuthology.orchestra.run.smithi153.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T03:49:11.181 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T03:49:11.362 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T03:49:11.445 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T03:49:11.458 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T03:49:11.495 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T03:49:11.749 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T03:49:11.763 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T03:49:11.807 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T03:49:11.815 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T03:49:11.831 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T03:49:11.873 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T03:49:12.127 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T03:49:12.140 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T03:49:12.168 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T03:49:12.201 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T03:49:12.216 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:12.259 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:12.496 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T03:49:12.509 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T03:49:12.546 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T03:49:12.899 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T03:49:12.914 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T03:49:12.982 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T03:49:13.310 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T03:49:13.326 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:13.360 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:14.692 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T03:49:14.709 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:14.817 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:15.314 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T03:49:15.330 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:15.372 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:15.861 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T03:49:15.877 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:15.973 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:16.497 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T03:49:16.511 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T03:49:16.556 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:22.282 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T03:49:22.299 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T03:49:22.340 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T03:49:22.643 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T03:49:22.657 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T03:49:22.701 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T03:49:23.079 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package thermald. 2023-10-09T03:49:23.093 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T03:49:23.160 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T03:49:23.624 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T03:49:23.641 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T03:49:23.682 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T03:49:23.709 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package upower. 2023-10-09T03:49:23.722 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T03:49:23.787 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T03:49:24.018 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T03:49:24.032 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T03:49:24.076 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T03:49:24.163 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T03:49:24.177 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T03:49:24.221 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T03:49:24.479 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package thermald. 2023-10-09T03:49:24.493 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T03:49:24.509 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T03:49:24.537 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T03:49:24.634 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T03:49:24.777 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T03:49:24.903 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T03:49:25.037 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T03:49:25.126 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package upower. 2023-10-09T03:49:25.140 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T03:49:25.163 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T03:49:25.196 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T03:49:25.289 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T03:49:25.860 INFO:teuthology.orchestra.run.smithi082.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T03:49:25.910 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T03:49:26.047 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T03:49:26.060 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T03:49:26.072 INFO:teuthology.orchestra.run.smithi082.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T03:49:26.077 INFO:teuthology.orchestra.run.smithi082.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T03:49:26.077 INFO:teuthology.orchestra.run.smithi082.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T03:49:26.105 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T03:49:26.429 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T03:49:26.584 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T03:49:26.594 INFO:teuthology.orchestra.run.smithi082.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T03:49:26.711 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T03:49:26.814 INFO:teuthology.orchestra.run.smithi082.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T03:49:26.836 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T03:49:26.857 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T03:49:26.987 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T03:49:27.105 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T03:49:27.218 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T03:49:27.218 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T03:49:27.256 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T03:49:27.685 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:27.812 INFO:teuthology.orchestra.run.smithi153.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T03:49:27.886 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T03:49:28.076 INFO:teuthology.orchestra.run.smithi153.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T03:49:28.083 INFO:teuthology.orchestra.run.smithi153.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T03:49:28.083 INFO:teuthology.orchestra.run.smithi153.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T03:49:28.619 INFO:teuthology.orchestra.run.smithi153.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T03:49:28.846 INFO:teuthology.orchestra.run.smithi153.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T03:49:28.890 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T03:49:29.024 INFO:teuthology.orchestra.run.smithi082.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:49:29.024 INFO:teuthology.orchestra.run.smithi082.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T03:49:29.189 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:29.198 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T03:49:29.198 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T03:49:29.653 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:30.615 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:31.014 INFO:teuthology.orchestra.run.smithi153.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:49:31.014 INFO:teuthology.orchestra.run.smithi153.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T03:49:31.198 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:31.983 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T03:49:32.108 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T03:49:32.590 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:33.581 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T03:49:33.991 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T03:49:34.125 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T03:49:34.359 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T03:49:34.416 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T03:49:34.451 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:34.534 INFO:teuthology.orchestra.run.smithi082.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T03:49:34.534 INFO:teuthology.orchestra.run.smithi082.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:49:35.219 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T03:49:35.319 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T03:49:35.394 INFO:teuthology.orchestra.run.smithi153.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T03:49:35.394 INFO:teuthology.orchestra.run.smithi153.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:49:59.989 INFO:teuthology.orchestra.run.smithi082.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T03:49:59.989 INFO:teuthology.orchestra.run.smithi082.stdout:Sourcing file `/etc/default/grub' 2023-10-09T03:49:59.998 INFO:teuthology.orchestra.run.smithi082.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T03:50:00.016 INFO:teuthology.orchestra.run.smithi082.stdout:Generating grub configuration file ... 2023-10-09T03:50:00.339 INFO:teuthology.orchestra.run.smithi082.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:50:00.376 INFO:teuthology.orchestra.run.smithi082.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:50:00.552 INFO:teuthology.orchestra.run.smithi153.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T03:50:00.552 INFO:teuthology.orchestra.run.smithi153.stdout:Sourcing file `/etc/default/grub' 2023-10-09T03:50:00.563 INFO:teuthology.orchestra.run.smithi153.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T03:50:00.580 INFO:teuthology.orchestra.run.smithi153.stdout:Generating grub configuration file ... 2023-10-09T03:50:00.753 INFO:teuthology.orchestra.run.smithi082.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T03:50:00.761 INFO:teuthology.orchestra.run.smithi082.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T03:50:00.900 INFO:teuthology.orchestra.run.smithi153.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:50:00.920 INFO:teuthology.orchestra.run.smithi153.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:50:01.342 INFO:teuthology.orchestra.run.smithi153.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T03:50:01.354 INFO:teuthology.orchestra.run.smithi153.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T03:50:02.810 INFO:teuthology.orchestra.run.smithi082.stdout:done 2023-10-09T03:50:03.223 DEBUG:teuthology.orchestra.run.smithi082:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T03:50:03.278 INFO:teuthology.orchestra.run.smithi082.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Status: install ok installed 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Priority: optional 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Section: kernel 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Installed-Size: 20 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T03:50:03.279 INFO:teuthology.orchestra.run.smithi082.stdout:Architecture: amd64 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Source: linux-meta-hwe-5.15 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Recommends: thermald 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout:Description: Generic Linux kernel image 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout: This package will always depend on the latest generic kernel image 2023-10-09T03:50:03.280 INFO:teuthology.orchestra.run.smithi082.stdout: available. 2023-10-09T03:50:03.280 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T03:50:03.281 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T03:50:03.281 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T03:50:03.281 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T03:50:03.281 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi082.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T03:50:03.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-get clean 2023-10-09T03:50:03.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-get update 2023-10-09T03:50:03.375 INFO:teuthology.orchestra.run.smithi153.stdout:done 2023-10-09T03:50:03.539 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:50:03.549 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T03:50:03.569 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T03:50:03.649 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T03:50:03.812 DEBUG:teuthology.orchestra.run.smithi153:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T03:50:03.858 INFO:teuthology.orchestra.run.smithi153.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Status: install ok installed 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Priority: optional 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Section: kernel 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Installed-Size: 20 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Architecture: amd64 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Source: linux-meta-hwe-5.15 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T03:50:03.859 INFO:teuthology.orchestra.run.smithi153.stdout:Recommends: thermald 2023-10-09T03:50:03.860 INFO:teuthology.orchestra.run.smithi153.stdout:Description: Generic Linux kernel image 2023-10-09T03:50:03.860 INFO:teuthology.orchestra.run.smithi153.stdout: This package will always depend on the latest generic kernel image 2023-10-09T03:50:03.860 INFO:teuthology.orchestra.run.smithi153.stdout: available. 2023-10-09T03:50:03.860 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T03:50:03.860 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T03:50:03.860 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T03:50:03.860 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T03:50:03.860 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi153.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T03:50:03.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-get clean 2023-10-09T03:50:03.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-get update 2023-10-09T03:50:04.205 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:50:04.205 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T03:50:04.276 INFO:teuthology.orchestra.run.smithi153.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T03:50:04.620 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T03:50:04.940 INFO:teuthology.orchestra.run.smithi153.stdout:Fetched 114 kB in 1s (141 kB/s) 2023-10-09T03:50:05.293 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:50:05.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T03:50:05.385 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:50:05.586 INFO:teuthology.orchestra.run.smithi082.stdout:Building dependency tree... 2023-10-09T03:50:05.587 INFO:teuthology.orchestra.run.smithi082.stdout:Reading state information... 2023-10-09T03:50:05.798 INFO:teuthology.orchestra.run.smithi082.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T03:50:05.798 INFO:teuthology.orchestra.run.smithi082.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T03:50:05.800 DEBUG:teuthology.orchestra.run.smithi082:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T03:50:05.815 INFO:teuthology.orchestra.run.smithi082.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T03:50:05.815 INFO:teuthology.orchestra.run.smithi082.stdout:Status: install ok installed 2023-10-09T03:50:05.815 INFO:teuthology.orchestra.run.smithi082.stdout:Priority: optional 2023-10-09T03:50:05.815 INFO:teuthology.orchestra.run.smithi082.stdout:Section: kernel 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Installed-Size: 20 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Architecture: amd64 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Source: linux-meta-hwe-5.15 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Recommends: thermald 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout:Description: Generic Linux kernel image 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout: This package will always depend on the latest generic kernel image 2023-10-09T03:50:05.816 INFO:teuthology.orchestra.run.smithi082.stdout: available. 2023-10-09T03:50:05.817 DEBUG:teuthology.orchestra.run.smithi082:> mktemp 2023-10-09T03:50:05.821 INFO:teuthology.orchestra.run.smithi082.stdout:/tmp/tmp.RszuVzmSST 2023-10-09T03:50:05.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo cp /boot/grub/grub.cfg /tmp/tmp.RszuVzmSST 2023-10-09T03:50:05.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 0666 /tmp/tmp.RszuVzmSST 2023-10-09T03:50:05.922 DEBUG:teuthology.orchestra.remote:smithi082:/tmp/tmp.RszuVzmSST is 9KB 2023-10-09T03:50:05.935 DEBUG:teuthology.orchestra.run.smithi082:> rm -fr /tmp/tmp.RszuVzmSST 2023-10-09T03:50:05.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T03:50:05.993 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:50:05.998 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:50:05.998 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T03:50:05.998 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T03:50:06.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T03:50:06.027 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T03:50:06.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo update-grub 2023-10-09T03:50:06.086 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:50:06.286 INFO:teuthology.orchestra.run.smithi153.stdout:Building dependency tree... 2023-10-09T03:50:06.286 INFO:teuthology.orchestra.run.smithi153.stdout:Reading state information... 2023-10-09T03:50:06.503 INFO:teuthology.orchestra.run.smithi153.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T03:50:06.503 INFO:teuthology.orchestra.run.smithi153.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T03:50:06.504 DEBUG:teuthology.orchestra.run.smithi153:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Status: install ok installed 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Priority: optional 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Section: kernel 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Installed-Size: 20 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Architecture: amd64 2023-10-09T03:50:06.520 INFO:teuthology.orchestra.run.smithi153.stdout:Source: linux-meta-hwe-5.15 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout:Recommends: thermald 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout:Description: Generic Linux kernel image 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout: This package will always depend on the latest generic kernel image 2023-10-09T03:50:06.521 INFO:teuthology.orchestra.run.smithi153.stdout: available. 2023-10-09T03:50:06.521 DEBUG:teuthology.orchestra.run.smithi153:> mktemp 2023-10-09T03:50:06.526 INFO:teuthology.orchestra.run.smithi153.stdout:/tmp/tmp.lQO2KoGLLL 2023-10-09T03:50:06.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo cp /boot/grub/grub.cfg /tmp/tmp.lQO2KoGLLL 2023-10-09T03:50:06.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 0666 /tmp/tmp.lQO2KoGLLL 2023-10-09T03:50:06.615 DEBUG:teuthology.orchestra.remote:smithi153:/tmp/tmp.lQO2KoGLLL is 9KB 2023-10-09T03:50:06.630 DEBUG:teuthology.orchestra.run.smithi153:> rm -fr /tmp/tmp.lQO2KoGLLL 2023-10-09T03:50:06.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T03:50:06.694 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:50:06.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T03:50:06.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T03:50:06.718 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T03:50:06.718 DEBUG:teuthology.orchestra.run.smithi153:> sudo update-grub 2023-10-09T03:50:07.296 INFO:teuthology.orchestra.run.smithi082.stderr:Sourcing file `/etc/default/grub' 2023-10-09T03:50:07.303 INFO:teuthology.orchestra.run.smithi082.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T03:50:07.314 INFO:teuthology.orchestra.run.smithi082.stderr:Generating grub configuration file ... 2023-10-09T03:50:07.568 INFO:teuthology.orchestra.run.smithi082.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:50:07.579 INFO:teuthology.orchestra.run.smithi082.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:50:07.976 INFO:teuthology.orchestra.run.smithi082.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T03:50:07.988 INFO:teuthology.orchestra.run.smithi082.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T03:50:08.053 INFO:teuthology.orchestra.run.smithi153.stderr:Sourcing file `/etc/default/grub' 2023-10-09T03:50:08.062 INFO:teuthology.orchestra.run.smithi153.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T03:50:08.073 INFO:teuthology.orchestra.run.smithi153.stderr:Generating grub configuration file ... 2023-10-09T03:50:08.356 INFO:teuthology.orchestra.run.smithi153.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T03:50:08.368 INFO:teuthology.orchestra.run.smithi153.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T03:50:08.761 INFO:teuthology.orchestra.run.smithi153.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T03:50:08.772 INFO:teuthology.orchestra.run.smithi153.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T03:50:08.998 INFO:teuthology.orchestra.run.smithi082.stderr:done 2023-10-09T03:50:09.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo shutdown -r now 2023-10-09T03:50:09.761 INFO:teuthology.orchestra.run.smithi153.stderr:done 2023-10-09T03:50:09.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo shutdown -r now 2023-10-09T03:50:39.030 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T03:50:39.458 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:50:39.458 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:50:39.766 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T03:50:39.766 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:50:39.767 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:50:57.914 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.82 2023-10-09T03:50:58.170 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.153 2023-10-09T03:51:06.922 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:51:06.923 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:51:07.172 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:51:07.172 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:51:09.978 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.82 2023-10-09T03:51:10.234 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.153 2023-10-09T03:51:21.990 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:51:21.991 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:51:22.235 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:51:22.236 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:51:40.410 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.82 2023-10-09T03:51:53.725 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.153 2023-10-09T03:51:55.411 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:51:55.413 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:51:55.826 DEBUG:teuthology.orchestra.run.smithi082:> true 2023-10-09T03:51:56.357 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:51:56.357 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-86-generic"... 2023-10-09T03:51:56.357 DEBUG:teuthology.orchestra.run.smithi082:> uname -r 2023-10-09T03:51:56.405 INFO:teuthology.orchestra.run.smithi082.stdout:5.15.0-86-generic 2023-10-09T03:51:56.405 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T03:51:56.406 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T03:51:56.406 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T03:51:57.407 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T03:51:57.407 DEBUG:teuthology.orchestra.run.smithi082:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:51:57.502 INFO:teuthology.orchestra.run.smithi082.stdout:ttyS1 2023-10-09T03:51:57.526 DEBUG:teuthology.parallel:result is None 2023-10-09T03:52:08.726 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:52:08.727 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:52:09.016 DEBUG:teuthology.orchestra.run.smithi153:> true 2023-10-09T03:52:09.523 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:52:09.523 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-86-generic"... 2023-10-09T03:52:09.523 DEBUG:teuthology.orchestra.run.smithi153:> uname -r 2023-10-09T03:52:09.573 INFO:teuthology.orchestra.run.smithi153.stdout:5.15.0-86-generic 2023-10-09T03:52:09.573 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T03:52:09.573 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T03:52:09.573 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T03:52:10.574 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T03:52:10.574 DEBUG:teuthology.orchestra.run.smithi153:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:52:10.669 INFO:teuthology.orchestra.run.smithi153.stdout:ttyS1 2023-10-09T03:52:10.697 DEBUG:teuthology.parallel:result is None 2023-10-09T03:52:10.697 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T03:52:10.704 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T03:52:10.704 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:52:10.707 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:52:10.714 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T03:52:10.720 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T03:52:10.730 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T03:52:10.731 DEBUG:teuthology.orchestra.run.smithi082:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:52:10.757 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:52:10.852 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T03:52:10.859 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T03:52:10.859 DEBUG:teuthology.orchestra.run.smithi082:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:52:10.861 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:52:10.876 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:52:10.879 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:52:10.884 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:52:10.885 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:52:10.887 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T03:52:10.893 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T03:52:10.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:52:10.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:52:10.946 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T03:52:10.955 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T03:52:10.955 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:52:10.989 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:52:10.997 DEBUG:teuthology.orchestra.run.smithi082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:52:11.042 DEBUG:teuthology.orchestra.run.smithi082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:52:11.090 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:52:11.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:52:11.155 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:52:11.162 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:52:11.213 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:52:11.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:52:11.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo service rsyslog restart 2023-10-09T03:52:11.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo service rsyslog restart 2023-10-09T03:52:11.364 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T03:52:11.371 INFO:teuthology.task.internal:Starting timer... 2023-10-09T03:52:11.371 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T03:52:11.384 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T03:52:11.392 DEBUG:teuthology.task.selinux:Excluding smithi082: OS 'ubuntu' does not support SELinux 2023-10-09T03:52:11.392 DEBUG:teuthology.task.selinux:Excluding smithi153: OS 'ubuntu' does not support SELinux 2023-10-09T03:52:11.392 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T03:52:11.392 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-09T03:52:11.392 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T03:52:11.393 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T03:52:11.402 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T03:52:11.407 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T03:52:11.533 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T03:52:11.548 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T03:52:11.549 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi082.front.sepia.ceph.com,smithi153.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T03:57:55.989 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi082.front.sepia.ceph.com'), Remote(name='ubuntu@smithi153.front.sepia.ceph.com')] 2023-10-09T03:57:56.020 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:57:56.021 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:57:56.101 DEBUG:teuthology.orchestra.run.smithi082:> true 2023-10-09T03:57:56.220 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi082.front.sepia.ceph.com' 2023-10-09T03:57:56.220 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:57:56.221 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:57:56.300 DEBUG:teuthology.orchestra.run.smithi153:> true 2023-10-09T03:57:56.416 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi153.front.sepia.ceph.com' 2023-10-09T03:57:56.416 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T03:57:56.426 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T03:57:56.426 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:57:56.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:57:56.429 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:57:56.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:57:56.458 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T03:57:56.458 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Command line: ntpd -gq 2023-10-09T03:57:56.459 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: proto: precision = 0.100 usec (-23) 2023-10-09T03:57:56.459 INFO:teuthology.orchestra.run.smithi082.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T03:57:56.460 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T03:57:56.460 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T03:57:56.460 INFO:teuthology.orchestra.run.smithi082.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T03:57:56.461 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T03:57:56.461 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T03:57:56.461 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T03:57:56.461 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen normally on 3 enp3s0f1 172.21.15.82:123 2023-10-09T03:57:56.461 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen normally on 4 lo [::1]:123 2023-10-09T03:57:56.462 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d3ff%5]:123 2023-10-09T03:57:56.462 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:57:56 ntpd[22683]: Listening on routing socket on fd #22 for interface updates 2023-10-09T03:57:56.487 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T03:57:56.488 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Command line: ntpd -gq 2023-10-09T03:57:56.488 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: proto: precision = 0.070 usec (-24) 2023-10-09T03:57:56.488 INFO:teuthology.orchestra.run.smithi153.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T03:57:56.489 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T03:57:56.489 INFO:teuthology.orchestra.run.smithi153.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T03:57:56.490 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T03:57:56.490 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen normally on 3 enp3s0f1 172.21.15.153:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen normally on 4 lo [::1]:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6deb%5]:123 2023-10-09T03:57:56.491 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:57:56 ntpd[22701]: Listening on routing socket on fd #22 for interface updates 2023-10-09T03:57:57.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:57.461 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:57 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:57.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:57 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:59.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:57:59 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:57:59.488 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:57:59.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:57:59.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:57:59.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:57:59 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:00.458 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:00.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:00 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:00.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:00.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:00.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:00.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:00.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:00 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:01.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:01.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:01 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:01.488 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.488 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:01.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:01.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:01.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:01 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:02.458 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:02.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:02 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:02.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:02.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:02 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:03.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:03.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:03.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:03.459 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stdout: 9 Oct 03:58:03 ntpd[22683]: ntpd: time slew +0.000497 s 2023-10-09T03:58:03.460 INFO:teuthology.orchestra.run.smithi082.stdout:ntpd: time slew +0.000497s 2023-10-09T03:58:03.461 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T03:58:03.461 INFO:teuthology.orchestra.run.smithi082.stderr: 9 Oct 03:58:03 ntpd[22683]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T03:58:03.489 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T03:58:03.490 INFO:teuthology.orchestra.run.smithi153.stdout: 9 Oct 03:58:03 ntpd[22701]: ntpd: time slew +0.001099 s 2023-10-09T03:58:03.490 INFO:teuthology.orchestra.run.smithi153.stdout:ntpd: time slew +0.001099s 2023-10-09T03:58:03.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T03:58:03.490 INFO:teuthology.orchestra.run.smithi153.stderr: 9 Oct 03:58:03 ntpd[22701]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T03:58:03.521 INFO:teuthology.orchestra.run.smithi082.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T03:58:03.521 INFO:teuthology.orchestra.run.smithi082.stdout:============================================================================== 2023-10-09T03:58:03.522 INFO:teuthology.orchestra.run.smithi082.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.522 INFO:teuthology.orchestra.run.smithi082.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.522 INFO:teuthology.orchestra.run.smithi082.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.522 INFO:teuthology.orchestra.run.smithi082.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.552 INFO:teuthology.orchestra.run.smithi153.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T03:58:03.552 INFO:teuthology.orchestra.run.smithi153.stdout:============================================================================== 2023-10-09T03:58:03.552 INFO:teuthology.orchestra.run.smithi153.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.553 INFO:teuthology.orchestra.run.smithi153.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.553 INFO:teuthology.orchestra.run.smithi153.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.553 INFO:teuthology.orchestra.run.smithi153.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T03:58:03.554 INFO:teuthology.run_tasks:Running task install... 2023-10-09T03:58:03.561 DEBUG:teuthology.task.install:project ceph 2023-10-09T03:58:03.562 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T03:58:03.562 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:58:03.562 INFO:teuthology.task.install:Using flavor: default 2023-10-09T03:58:03.567 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T03:58:03.567 INFO:teuthology.task.install:extra packages: [] 2023-10-09T03:58:03.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-key list | grep Ceph 2023-10-09T03:58:03.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-key list | grep Ceph 2023-10-09T03:58:03.676 INFO:teuthology.orchestra.run.smithi082.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T03:58:03.693 INFO:teuthology.orchestra.run.smithi153.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T03:58:03.732 INFO:teuthology.orchestra.run.smithi082.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T03:58:03.732 INFO:teuthology.orchestra.run.smithi082.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T03:58:03.733 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T03:58:03.733 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:58:03.733 INFO:teuthology.packaging:ref: None 2023-10-09T03:58:03.733 INFO:teuthology.packaging:tag: None 2023-10-09T03:58:03.733 INFO:teuthology.packaging:branch: pacific 2023-10-09T03:58:03.733 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:58:03.734 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T03:58:03.748 INFO:teuthology.orchestra.run.smithi153.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T03:58:03.748 INFO:teuthology.orchestra.run.smithi153.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T03:58:03.749 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T03:58:03.749 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:58:03.749 INFO:teuthology.packaging:ref: None 2023-10-09T03:58:03.749 INFO:teuthology.packaging:tag: None 2023-10-09T03:58:03.749 INFO:teuthology.packaging:branch: pacific 2023-10-09T03:58:03.750 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:58:03.750 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T03:58:03.969 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T03:58:03.970 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T03:58:04.048 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T03:58:04.048 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T03:58:04.077 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:58:04.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T03:58:04.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo apt-get update 2023-10-09T03:58:04.216 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:58:04.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T03:58:04.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo apt-get update 2023-10-09T03:58:04.305 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:58:04.306 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T03:58:04.313 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T03:58:04.332 INFO:teuthology.orchestra.run.smithi153.stdout:Ign:4 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T03:58:04.333 INFO:teuthology.orchestra.run.smithi153.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T03:58:04.353 INFO:teuthology.orchestra.run.smithi153.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T03:58:04.376 INFO:teuthology.orchestra.run.smithi153.stdout:Ign:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T03:58:04.397 INFO:teuthology.orchestra.run.smithi153.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T03:58:04.432 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T03:58:04.442 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T03:58:04.443 INFO:teuthology.orchestra.run.smithi153.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T03:58:04.462 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T03:58:04.482 INFO:teuthology.orchestra.run.smithi082.stdout:Ign:4 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T03:58:04.503 INFO:teuthology.orchestra.run.smithi082.stdout:Get:5 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T03:58:04.526 INFO:teuthology.orchestra.run.smithi082.stdout:Ign:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T03:58:04.546 INFO:teuthology.orchestra.run.smithi082.stdout:Hit:7 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T03:58:04.547 INFO:teuthology.orchestra.run.smithi082.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T03:58:04.584 INFO:teuthology.orchestra.run.smithi082.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T03:58:04.925 INFO:teuthology.orchestra.run.smithi153.stdout:Fetched 28.0 kB in 1s (44.8 kB/s) 2023-10-09T03:58:05.045 INFO:teuthology.orchestra.run.smithi082.stdout:Fetched 28.0 kB in 1s (46.7 kB/s) 2023-10-09T03:58:05.976 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:58:06.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T03:58:06.078 INFO:teuthology.orchestra.run.smithi153.stdout:Reading package lists... 2023-10-09T03:58:06.092 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:58:06.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T03:58:06.193 INFO:teuthology.orchestra.run.smithi082.stdout:Reading package lists... 2023-10-09T03:58:06.279 INFO:teuthology.orchestra.run.smithi153.stdout:Building dependency tree... 2023-10-09T03:58:06.279 INFO:teuthology.orchestra.run.smithi153.stdout:Reading state information... 2023-10-09T03:58:06.372 INFO:teuthology.orchestra.run.smithi082.stdout:Building dependency tree... 2023-10-09T03:58:06.373 INFO:teuthology.orchestra.run.smithi082.stdout:Reading state information... 2023-10-09T03:58:06.457 INFO:teuthology.orchestra.run.smithi153.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T03:58:06.458 INFO:teuthology.orchestra.run.smithi153.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T03:58:06.458 INFO:teuthology.orchestra.run.smithi153.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T03:58:06.459 INFO:teuthology.orchestra.run.smithi153.stdout:The following additional packages will be installed: 2023-10-09T03:58:06.459 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T03:58:06.459 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T03:58:06.460 INFO:teuthology.orchestra.run.smithi153.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout:Suggested packages: 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T03:58:06.461 INFO:teuthology.orchestra.run.smithi153.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout:Recommended packages: 2023-10-09T03:58:06.462 INFO:teuthology.orchestra.run.smithi153.stdout: btrfs-tools 2023-10-09T03:58:06.539 INFO:teuthology.orchestra.run.smithi082.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T03:58:06.540 INFO:teuthology.orchestra.run.smithi082.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T03:58:06.540 INFO:teuthology.orchestra.run.smithi082.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T03:58:06.540 INFO:teuthology.orchestra.run.smithi082.stdout:The following additional packages will be installed: 2023-10-09T03:58:06.540 INFO:teuthology.orchestra.run.smithi082.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T03:58:06.541 INFO:teuthology.orchestra.run.smithi082.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T03:58:06.541 INFO:teuthology.orchestra.run.smithi082.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T03:58:06.541 INFO:teuthology.orchestra.run.smithi082.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T03:58:06.542 INFO:teuthology.orchestra.run.smithi082.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout:Suggested packages: 2023-10-09T03:58:06.543 INFO:teuthology.orchestra.run.smithi082.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout:Recommended packages: 2023-10-09T03:58:06.544 INFO:teuthology.orchestra.run.smithi082.stdout: btrfs-tools 2023-10-09T03:58:06.570 INFO:teuthology.orchestra.run.smithi153.stdout:The following NEW packages will be installed: 2023-10-09T03:58:06.570 INFO:teuthology.orchestra.run.smithi153.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T03:58:06.571 INFO:teuthology.orchestra.run.smithi153.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T03:58:06.572 INFO:teuthology.orchestra.run.smithi153.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T03:58:06.573 INFO:teuthology.orchestra.run.smithi153.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T03:58:06.573 INFO:teuthology.orchestra.run.smithi153.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T03:58:06.573 INFO:teuthology.orchestra.run.smithi153.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T03:58:06.573 INFO:teuthology.orchestra.run.smithi153.stdout:The following packages will be upgraded: 2023-10-09T03:58:06.573 INFO:teuthology.orchestra.run.smithi153.stdout: librados2 librbd1 2023-10-09T03:58:06.636 INFO:teuthology.orchestra.run.smithi153.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T03:58:06.636 INFO:teuthology.orchestra.run.smithi153.stdout:Need to get 174 MB of archives. 2023-10-09T03:58:06.637 INFO:teuthology.orchestra.run.smithi153.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T03:58:06.637 INFO:teuthology.orchestra.run.smithi153.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout:The following NEW packages will be installed: 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T03:58:06.651 INFO:teuthology.orchestra.run.smithi082.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T03:58:06.652 INFO:teuthology.orchestra.run.smithi082.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T03:58:06.652 INFO:teuthology.orchestra.run.smithi082.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T03:58:06.652 INFO:teuthology.orchestra.run.smithi082.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T03:58:06.652 INFO:teuthology.orchestra.run.smithi082.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T03:58:06.652 INFO:teuthology.orchestra.run.smithi082.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T03:58:06.653 INFO:teuthology.orchestra.run.smithi082.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T03:58:06.654 INFO:teuthology.orchestra.run.smithi082.stdout:The following packages will be upgraded: 2023-10-09T03:58:06.654 INFO:teuthology.orchestra.run.smithi082.stdout: librados2 librbd1 2023-10-09T03:58:06.716 INFO:teuthology.orchestra.run.smithi082.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T03:58:06.717 INFO:teuthology.orchestra.run.smithi082.stdout:Need to get 174 MB of archives. 2023-10-09T03:58:06.717 INFO:teuthology.orchestra.run.smithi082.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T03:58:06.717 INFO:teuthology.orchestra.run.smithi082.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T03:58:06.718 INFO:teuthology.orchestra.run.smithi153.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T03:58:06.719 INFO:teuthology.orchestra.run.smithi153.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T03:58:06.729 INFO:teuthology.orchestra.run.smithi153.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T03:58:06.735 INFO:teuthology.orchestra.run.smithi153.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T03:58:06.758 INFO:teuthology.orchestra.run.smithi153.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T03:58:06.760 INFO:teuthology.orchestra.run.smithi153.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T03:58:06.762 INFO:teuthology.orchestra.run.smithi153.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T03:58:06.763 INFO:teuthology.orchestra.run.smithi153.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T03:58:06.770 INFO:teuthology.orchestra.run.smithi153.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T03:58:06.771 INFO:teuthology.orchestra.run.smithi153.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T03:58:06.772 INFO:teuthology.orchestra.run.smithi153.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T03:58:06.775 INFO:teuthology.orchestra.run.smithi153.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T03:58:06.776 INFO:teuthology.orchestra.run.smithi153.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T03:58:06.776 INFO:teuthology.orchestra.run.smithi153.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T03:58:06.779 INFO:teuthology.orchestra.run.smithi082.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T03:58:06.788 INFO:teuthology.orchestra.run.smithi153.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T03:58:06.790 INFO:teuthology.orchestra.run.smithi153.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T03:58:06.791 INFO:teuthology.orchestra.run.smithi153.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T03:58:06.792 INFO:teuthology.orchestra.run.smithi153.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T03:58:06.797 INFO:teuthology.orchestra.run.smithi082.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T03:58:06.799 INFO:teuthology.orchestra.run.smithi082.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T03:58:06.805 INFO:teuthology.orchestra.run.smithi153.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T03:58:06.805 INFO:teuthology.orchestra.run.smithi153.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T03:58:06.806 INFO:teuthology.orchestra.run.smithi153.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T03:58:06.808 INFO:teuthology.orchestra.run.smithi153.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T03:58:06.808 INFO:teuthology.orchestra.run.smithi082.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T03:58:06.808 INFO:teuthology.orchestra.run.smithi153.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T03:58:06.813 INFO:teuthology.orchestra.run.smithi153.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T03:58:06.814 INFO:teuthology.orchestra.run.smithi082.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T03:58:06.822 INFO:teuthology.orchestra.run.smithi153.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T03:58:06.822 INFO:teuthology.orchestra.run.smithi153.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T03:58:06.823 INFO:teuthology.orchestra.run.smithi082.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T03:58:06.823 INFO:teuthology.orchestra.run.smithi153.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T03:58:06.825 INFO:teuthology.orchestra.run.smithi153.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T03:58:06.834 INFO:teuthology.orchestra.run.smithi153.stdout:Get:30 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T03:58:06.837 INFO:teuthology.orchestra.run.smithi082.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T03:58:06.838 INFO:teuthology.orchestra.run.smithi153.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T03:58:06.840 INFO:teuthology.orchestra.run.smithi153.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T03:58:06.841 INFO:teuthology.orchestra.run.smithi153.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T03:58:06.841 INFO:teuthology.orchestra.run.smithi082.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T03:58:06.843 INFO:teuthology.orchestra.run.smithi082.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T03:58:06.843 INFO:teuthology.orchestra.run.smithi153.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T03:58:06.845 INFO:teuthology.orchestra.run.smithi153.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T03:58:06.847 INFO:teuthology.orchestra.run.smithi153.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T03:58:06.849 INFO:teuthology.orchestra.run.smithi082.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T03:58:06.850 INFO:teuthology.orchestra.run.smithi082.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T03:58:06.851 INFO:teuthology.orchestra.run.smithi082.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T03:58:06.858 INFO:teuthology.orchestra.run.smithi082.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T03:58:06.858 INFO:teuthology.orchestra.run.smithi153.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T03:58:06.858 INFO:teuthology.orchestra.run.smithi082.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T03:58:06.859 INFO:teuthology.orchestra.run.smithi153.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T03:58:06.859 INFO:teuthology.orchestra.run.smithi082.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T03:58:06.868 INFO:teuthology.orchestra.run.smithi082.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T03:58:06.869 INFO:teuthology.orchestra.run.smithi082.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T03:58:06.869 INFO:teuthology.orchestra.run.smithi082.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T03:58:06.870 INFO:teuthology.orchestra.run.smithi082.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T03:58:06.884 INFO:teuthology.orchestra.run.smithi082.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T03:58:06.885 INFO:teuthology.orchestra.run.smithi082.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T03:58:06.886 INFO:teuthology.orchestra.run.smithi082.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T03:58:06.887 INFO:teuthology.orchestra.run.smithi082.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T03:58:06.888 INFO:teuthology.orchestra.run.smithi082.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T03:58:06.893 INFO:teuthology.orchestra.run.smithi082.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T03:58:06.902 INFO:teuthology.orchestra.run.smithi082.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T03:58:06.902 INFO:teuthology.orchestra.run.smithi082.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T03:58:06.902 INFO:teuthology.orchestra.run.smithi082.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T03:58:06.903 INFO:teuthology.orchestra.run.smithi082.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T03:58:06.918 INFO:teuthology.orchestra.run.smithi082.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T03:58:06.920 INFO:teuthology.orchestra.run.smithi082.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T03:58:06.921 INFO:teuthology.orchestra.run.smithi082.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T03:58:06.923 INFO:teuthology.orchestra.run.smithi082.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T03:58:06.925 INFO:teuthology.orchestra.run.smithi082.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T03:58:06.928 INFO:teuthology.orchestra.run.smithi082.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T03:58:06.934 INFO:teuthology.orchestra.run.smithi082.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T03:58:06.935 INFO:teuthology.orchestra.run.smithi082.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T03:58:06.970 INFO:teuthology.orchestra.run.smithi153.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T03:58:06.986 INFO:teuthology.orchestra.run.smithi153.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T03:58:06.987 INFO:teuthology.orchestra.run.smithi153.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T03:58:06.999 INFO:teuthology.orchestra.run.smithi153.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T03:58:06.999 INFO:teuthology.orchestra.run.smithi153.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T03:58:06.999 INFO:teuthology.orchestra.run.smithi153.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T03:58:06.999 INFO:teuthology.orchestra.run.smithi153.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi153.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi153.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi153.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T03:58:07.006 INFO:teuthology.orchestra.run.smithi153.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T03:58:07.007 INFO:teuthology.orchestra.run.smithi153.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T03:58:07.024 INFO:teuthology.orchestra.run.smithi153.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T03:58:07.029 INFO:teuthology.orchestra.run.smithi153.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T03:58:07.032 INFO:teuthology.orchestra.run.smithi153.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T03:58:07.033 INFO:teuthology.orchestra.run.smithi153.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T03:58:07.034 INFO:teuthology.orchestra.run.smithi153.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T03:58:07.037 INFO:teuthology.orchestra.run.smithi153.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T03:58:07.039 INFO:teuthology.orchestra.run.smithi153.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T03:58:07.042 INFO:teuthology.orchestra.run.smithi153.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T03:58:07.042 INFO:teuthology.orchestra.run.smithi153.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T03:58:07.042 INFO:teuthology.orchestra.run.smithi153.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T03:58:07.057 INFO:teuthology.orchestra.run.smithi153.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T03:58:07.059 INFO:teuthology.orchestra.run.smithi153.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T03:58:07.072 INFO:teuthology.orchestra.run.smithi082.stdout:Get:39 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T03:58:07.073 INFO:teuthology.orchestra.run.smithi082.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T03:58:07.076 INFO:teuthology.orchestra.run.smithi153.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T03:58:07.076 INFO:teuthology.orchestra.run.smithi153.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T03:58:07.077 INFO:teuthology.orchestra.run.smithi153.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T03:58:07.077 INFO:teuthology.orchestra.run.smithi153.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T03:58:07.078 INFO:teuthology.orchestra.run.smithi153.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T03:58:07.079 INFO:teuthology.orchestra.run.smithi153.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T03:58:07.079 INFO:teuthology.orchestra.run.smithi153.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T03:58:07.079 INFO:teuthology.orchestra.run.smithi153.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T03:58:07.088 INFO:teuthology.orchestra.run.smithi082.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T03:58:07.089 INFO:teuthology.orchestra.run.smithi082.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T03:58:07.090 INFO:teuthology.orchestra.run.smithi153.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T03:58:07.101 INFO:teuthology.orchestra.run.smithi082.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T03:58:07.101 INFO:teuthology.orchestra.run.smithi082.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T03:58:07.102 INFO:teuthology.orchestra.run.smithi082.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T03:58:07.102 INFO:teuthology.orchestra.run.smithi082.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T03:58:07.103 INFO:teuthology.orchestra.run.smithi082.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T03:58:07.103 INFO:teuthology.orchestra.run.smithi082.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T03:58:07.103 INFO:teuthology.orchestra.run.smithi082.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T03:58:07.106 INFO:teuthology.orchestra.run.smithi082.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T03:58:07.121 INFO:teuthology.orchestra.run.smithi082.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T03:58:07.122 INFO:teuthology.orchestra.run.smithi082.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T03:58:07.128 INFO:teuthology.orchestra.run.smithi082.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T03:58:07.131 INFO:teuthology.orchestra.run.smithi082.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T03:58:07.134 INFO:teuthology.orchestra.run.smithi082.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T03:58:07.134 INFO:teuthology.orchestra.run.smithi082.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T03:58:07.136 INFO:teuthology.orchestra.run.smithi082.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T03:58:07.138 INFO:teuthology.orchestra.run.smithi082.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T03:58:07.141 INFO:teuthology.orchestra.run.smithi082.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T03:58:07.141 INFO:teuthology.orchestra.run.smithi082.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T03:58:07.150 INFO:teuthology.orchestra.run.smithi082.stdout:Get:61 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T03:58:07.157 INFO:teuthology.orchestra.run.smithi082.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T03:58:07.157 INFO:teuthology.orchestra.run.smithi082.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T03:58:07.159 INFO:teuthology.orchestra.run.smithi082.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T03:58:07.177 INFO:teuthology.orchestra.run.smithi082.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T03:58:07.177 INFO:teuthology.orchestra.run.smithi082.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T03:58:07.177 INFO:teuthology.orchestra.run.smithi082.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T03:58:07.178 INFO:teuthology.orchestra.run.smithi082.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T03:58:07.178 INFO:teuthology.orchestra.run.smithi082.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T03:58:07.179 INFO:teuthology.orchestra.run.smithi082.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T03:58:07.180 INFO:teuthology.orchestra.run.smithi082.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T03:58:07.180 INFO:teuthology.orchestra.run.smithi082.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T03:58:07.190 INFO:teuthology.orchestra.run.smithi082.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T03:58:07.192 INFO:teuthology.orchestra.run.smithi082.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T03:58:07.193 INFO:teuthology.orchestra.run.smithi082.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T03:58:07.194 INFO:teuthology.orchestra.run.smithi082.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T03:58:07.201 INFO:teuthology.orchestra.run.smithi082.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T03:58:07.202 INFO:teuthology.orchestra.run.smithi082.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T03:58:07.218 INFO:teuthology.orchestra.run.smithi082.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T03:58:07.363 INFO:teuthology.orchestra.run.smithi082.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T03:58:07.366 INFO:teuthology.orchestra.run.smithi082.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T03:58:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T03:58:07.590 INFO:teuthology.orchestra.run.smithi153.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T03:58:08.217 INFO:teuthology.orchestra.run.smithi153.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T03:58:08.351 INFO:teuthology.orchestra.run.smithi153.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T03:58:08.379 INFO:teuthology.orchestra.run.smithi082.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T03:58:08.411 INFO:teuthology.orchestra.run.smithi153.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T03:58:08.412 INFO:teuthology.orchestra.run.smithi153.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T03:58:08.453 INFO:teuthology.orchestra.run.smithi153.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T03:58:08.456 INFO:teuthology.orchestra.run.smithi153.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T03:58:08.526 INFO:teuthology.orchestra.run.smithi082.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T03:58:08.538 INFO:teuthology.orchestra.run.smithi082.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T03:58:08.538 INFO:teuthology.orchestra.run.smithi153.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T03:58:08.542 INFO:teuthology.orchestra.run.smithi082.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T03:58:08.602 INFO:teuthology.orchestra.run.smithi082.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T03:58:08.882 INFO:teuthology.orchestra.run.smithi082.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T03:58:09.202 INFO:teuthology.orchestra.run.smithi153.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T03:58:09.223 INFO:teuthology.orchestra.run.smithi153.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T03:58:09.312 INFO:teuthology.orchestra.run.smithi153.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T03:58:10.039 INFO:teuthology.orchestra.run.smithi082.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T03:58:10.039 INFO:teuthology.orchestra.run.smithi082.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T03:58:10.052 INFO:teuthology.orchestra.run.smithi082.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T03:58:10.069 INFO:teuthology.orchestra.run.smithi082.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T03:58:10.069 INFO:teuthology.orchestra.run.smithi082.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T03:58:10.070 INFO:teuthology.orchestra.run.smithi082.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T03:58:10.154 INFO:teuthology.orchestra.run.smithi082.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T03:58:10.465 INFO:teuthology.orchestra.run.smithi082.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T03:58:10.465 INFO:teuthology.orchestra.run.smithi082.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T03:58:12.675 INFO:teuthology.orchestra.run.smithi082.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T03:58:12.675 INFO:teuthology.orchestra.run.smithi082.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T03:58:12.970 INFO:teuthology.orchestra.run.smithi082.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T03:58:13.297 INFO:teuthology.orchestra.run.smithi153.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T03:58:13.622 INFO:teuthology.orchestra.run.smithi082.stdout:Fetched 174 MB in 6s (27.5 MB/s) 2023-10-09T03:58:13.699 INFO:teuthology.orchestra.run.smithi153.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T03:58:13.717 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libjaeger. 2023-10-09T03:58:13.727 INFO:teuthology.orchestra.run.smithi153.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T03:58:13.730 INFO:teuthology.orchestra.run.smithi153.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T03:58:13.763 INFO:teuthology.orchestra.run.smithi082.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T03:58:13.766 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:13.815 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:13.842 INFO:teuthology.orchestra.run.smithi153.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T03:58:14.143 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T03:58:14.156 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T03:58:14.235 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T03:58:14.378 INFO:teuthology.orchestra.run.smithi153.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T03:58:14.603 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:14.730 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T03:58:15.382 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:15.510 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T03:58:16.155 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T03:58:16.164 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:16.205 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:16.508 INFO:teuthology.orchestra.run.smithi153.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T03:58:16.508 INFO:teuthology.orchestra.run.smithi153.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T03:58:16.515 INFO:teuthology.orchestra.run.smithi153.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T03:58:16.535 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-rados. 2023-10-09T03:58:16.546 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:16.591 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:16.692 INFO:teuthology.orchestra.run.smithi153.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T03:58:16.694 INFO:teuthology.orchestra.run.smithi153.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T03:58:16.724 INFO:teuthology.orchestra.run.smithi153.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T03:58:16.938 INFO:teuthology.orchestra.run.smithi153.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T03:58:16.945 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T03:58:16.961 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:17.004 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:17.349 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T03:58:17.362 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:17.407 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:17.512 INFO:teuthology.orchestra.run.smithi153.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T03:58:17.513 INFO:teuthology.orchestra.run.smithi153.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T03:58:17.743 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T03:58:17.758 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:17.802 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:18.135 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T03:58:18.149 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T03:58:18.185 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T03:58:18.421 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T03:58:18.434 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:18.471 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:18.799 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T03:58:18.813 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T03:58:18.858 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T03:58:19.169 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T03:58:19.182 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T03:58:19.219 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T03:58:19.547 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T03:58:19.560 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T03:58:19.605 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T03:58:19.949 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package librgw2. 2023-10-09T03:58:19.962 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:20.007 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:20.620 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T03:58:20.634 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:20.677 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:20.972 INFO:teuthology.orchestra.run.smithi153.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T03:58:20.972 INFO:teuthology.orchestra.run.smithi153.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T03:58:21.005 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T03:58:21.013 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T03:58:21.055 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T03:58:21.399 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package liboath0. 2023-10-09T03:58:21.408 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T03:58:21.449 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T03:58:21.465 INFO:teuthology.orchestra.run.smithi153.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T03:58:21.760 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T03:58:21.773 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:21.810 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:22.114 INFO:teuthology.orchestra.run.smithi153.stdout:Fetched 174 MB in 15s (11.7 MB/s) 2023-10-09T03:58:22.242 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libjaeger. 2023-10-09T03:58:22.247 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-common. 2023-10-09T03:58:22.260 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:22.289 INFO:teuthology.orchestra.run.smithi153.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T03:58:22.292 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:22.305 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:22.340 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:22.734 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T03:58:22.748 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T03:58:22.851 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T03:58:23.212 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:23.339 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T03:58:24.057 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:24.209 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T03:58:24.535 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-base. 2023-10-09T03:58:24.543 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:24.595 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:25.099 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T03:58:25.112 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:25.157 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:25.533 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T03:58:25.535 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-rados. 2023-10-09T03:58:25.546 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T03:58:25.549 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:25.591 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T03:58:25.602 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:25.986 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T03:58:26.000 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T03:58:26.027 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T03:58:26.041 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:26.044 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T03:58:26.093 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:26.289 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T03:58:26.306 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:26.332 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:26.422 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T03:58:26.435 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:26.471 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:26.694 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T03:58:26.710 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T03:58:26.752 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T03:58:26.783 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T03:58:26.796 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:26.833 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:27.055 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T03:58:27.070 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T03:58:27.113 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T03:58:27.144 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T03:58:27.157 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T03:58:27.194 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T03:58:27.441 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T03:58:27.447 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T03:58:27.454 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T03:58:27.461 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:27.491 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T03:58:27.505 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:27.803 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T03:58:27.818 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T03:58:27.824 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T03:58:27.838 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T03:58:27.861 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T03:58:27.899 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T03:58:28.164 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T03:58:28.178 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T03:58:28.222 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T03:58:28.245 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T03:58:28.260 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T03:58:28.303 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T03:58:28.534 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-mako. 2023-10-09T03:58:28.550 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T03:58:28.609 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T03:58:28.648 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T03:58:28.661 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T03:58:28.697 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T03:58:28.920 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T03:58:28.933 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T03:58:28.970 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T03:58:29.092 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package librgw2. 2023-10-09T03:58:29.105 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:29.158 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:29.290 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T03:58:29.305 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T03:58:29.348 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T03:58:29.651 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-webob. 2023-10-09T03:58:29.664 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T03:58:29.701 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T03:58:29.753 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T03:58:29.766 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:29.802 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:30.029 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T03:58:30.041 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T03:58:30.078 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T03:58:30.131 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T03:58:30.145 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T03:58:30.180 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T03:58:30.390 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T03:58:30.405 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T03:58:30.448 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T03:58:30.525 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package liboath0. 2023-10-09T03:58:30.540 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T03:58:30.583 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T03:58:30.793 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T03:58:30.807 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T03:58:30.854 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T03:58:30.894 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T03:58:30.909 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:30.952 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:31.171 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T03:58:31.184 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T03:58:31.229 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T03:58:31.441 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-common. 2023-10-09T03:58:31.456 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:31.507 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:31.557 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-paste. 2023-10-09T03:58:31.571 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T03:58:31.615 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T03:58:31.935 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T03:58:31.952 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T03:58:31.985 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T03:58:32.279 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T03:58:32.293 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T03:58:32.338 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T03:58:32.657 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T03:58:32.670 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T03:58:32.715 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T03:58:33.027 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T03:58:33.040 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T03:58:33.085 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T03:58:33.431 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T03:58:33.439 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T03:58:33.481 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T03:58:33.755 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-base. 2023-10-09T03:58:33.763 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:33.814 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:33.868 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T03:58:33.876 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:33.917 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:34.312 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T03:58:34.325 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:34.362 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:34.742 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T03:58:34.756 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T03:58:34.792 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T03:58:34.849 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T03:58:34.862 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:34.907 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:35.190 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T03:58:35.205 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T03:58:35.247 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T03:58:35.500 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T03:58:35.514 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:35.558 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:35.922 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T03:58:35.935 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T03:58:35.971 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T03:58:35.985 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T03:58:35.994 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:36.035 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:36.266 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T03:58:36.281 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T03:58:36.324 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T03:58:36.694 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T03:58:36.709 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T03:58:36.752 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T03:58:37.063 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T03:58:37.078 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T03:58:37.122 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T03:58:37.400 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T03:58:37.413 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T03:58:37.449 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T03:58:37.752 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-mako. 2023-10-09T03:58:37.766 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T03:58:37.810 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T03:58:38.114 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T03:58:38.127 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T03:58:38.172 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T03:58:38.474 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T03:58:38.489 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T03:58:38.532 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T03:58:38.640 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph. 2023-10-09T03:58:38.654 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:38.698 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:38.835 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-webob. 2023-10-09T03:58:38.849 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T03:58:38.893 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T03:58:38.951 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T03:58:38.965 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:39.009 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:39.196 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T03:58:39.210 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T03:58:39.246 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T03:58:39.549 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T03:58:39.565 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T03:58:39.615 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T03:58:39.943 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T03:58:39.956 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T03:58:39.996 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T03:58:40.122 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T03:58:40.137 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:40.188 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:40.700 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package cephadm. 2023-10-09T03:58:40.713 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:40.749 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:40.980 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T03:58:40.994 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T03:58:41.038 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T03:58:41.069 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T03:58:41.082 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T03:58:41.119 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T03:58:41.324 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-paste. 2023-10-09T03:58:41.340 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T03:58:41.383 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T03:58:41.430 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T03:58:41.445 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:41.488 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:41.711 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T03:58:41.725 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T03:58:41.769 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T03:58:41.816 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T03:58:41.825 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T03:58:41.866 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T03:58:42.071 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T03:58:42.085 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T03:58:42.119 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T03:58:42.129 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T03:58:42.132 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T03:58:42.160 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T03:58:42.408 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T03:58:42.421 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T03:58:42.422 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T03:58:42.434 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T03:58:42.466 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T03:58:42.471 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T03:58:42.769 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T03:58:42.782 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T03:58:42.807 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-routes. 2023-10-09T03:58:42.820 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T03:58:42.827 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T03:58:42.865 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T03:58:43.190 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T03:58:43.202 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T03:58:43.203 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T03:58:43.215 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:43.248 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T03:58:43.260 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:43.584 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T03:58:43.598 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:43.643 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:44.004 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T03:58:44.018 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:44.062 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:44.209 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T03:58:44.223 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T03:58:44.267 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T03:58:44.516 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T03:58:44.529 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:44.570 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T03:58:44.573 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:44.584 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T03:58:44.628 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T03:58:44.914 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T03:58:44.930 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T03:58:44.972 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T03:58:45.652 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T03:58:45.666 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:45.710 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:46.290 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T03:58:46.306 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T03:58:46.348 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T03:58:46.878 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T03:58:46.894 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T03:58:46.936 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T03:58:47.307 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T03:58:47.321 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T03:58:47.365 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T03:58:47.989 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T03:58:47.998 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:48.039 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:48.316 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph. 2023-10-09T03:58:48.330 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:48.373 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:48.693 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T03:58:48.708 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:48.751 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:49.167 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T03:58:49.179 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T03:58:49.182 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T03:58:49.187 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:49.225 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T03:58:49.229 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:49.528 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T03:58:49.543 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T03:58:49.586 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T03:58:49.757 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package cephadm. 2023-10-09T03:58:49.765 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:49.806 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:49.923 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T03:58:49.936 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T03:58:49.981 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T03:58:50.101 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T03:58:50.114 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T03:58:50.159 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T03:58:50.275 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T03:58:50.290 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T03:58:50.333 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T03:58:50.462 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T03:58:50.477 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:50.520 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:50.645 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T03:58:50.659 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T03:58:50.703 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T03:58:50.873 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T03:58:50.887 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T03:58:50.931 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T03:58:51.023 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T03:58:51.036 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T03:58:51.102 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T03:58:51.176 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T03:58:51.189 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T03:58:51.234 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T03:58:51.384 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T03:58:51.397 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T03:58:51.442 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T03:58:51.486 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T03:58:51.501 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T03:58:51.544 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T03:58:51.745 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T03:58:51.759 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T03:58:51.803 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T03:58:51.847 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-routes. 2023-10-09T03:58:51.861 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T03:58:51.905 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T03:58:52.115 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T03:58:52.128 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T03:58:52.173 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T03:58:52.258 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T03:58:52.271 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:52.316 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:52.492 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T03:58:52.506 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T03:58:52.550 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T03:58:53.072 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T03:58:53.086 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:53.130 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:53.265 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T03:58:53.281 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T03:58:53.323 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T03:58:53.449 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T03:58:53.464 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T03:58:53.507 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T03:58:53.609 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T03:58:53.624 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T03:58:53.667 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T03:58:53.852 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T03:58:53.867 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T03:58:53.952 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:58:53.969 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T03:58:53.983 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T03:58:54.028 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T03:58:54.205 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package jq. 2023-10-09T03:58:54.220 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T03:58:54.262 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:58:54.515 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package socat. 2023-10-09T03:58:54.529 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T03:58:54.573 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T03:58:54.885 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T03:58:54.899 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T03:58:54.951 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T03:58:55.311 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T03:58:55.326 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T03:58:55.369 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T03:58:55.373 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package ceph-test. 2023-10-09T03:58:55.387 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:55.446 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:55.915 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T03:58:55.928 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T03:58:55.973 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T03:58:56.685 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T03:58:56.702 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T03:58:56.742 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T03:58:57.501 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T03:58:57.515 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:58:57.559 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:58.703 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T03:58:58.717 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T03:58:58.761 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T03:58:59.064 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T03:58:59.078 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T03:58:59.122 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T03:58:59.442 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T03:58:59.457 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T03:58:59.500 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T03:58:59.751 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T03:58:59.767 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:58:59.818 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:58:59.819 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T03:58:59.833 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T03:58:59.877 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T03:59:00.189 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T03:59:00.203 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T03:59:00.221 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T03:59:00.230 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T03:59:00.247 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T03:59:00.287 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T03:59:00.575 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T03:59:00.590 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T03:59:00.633 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T03:59:00.653 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T03:59:00.662 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T03:59:00.695 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T03:59:00.936 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T03:59:00.950 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T03:59:00.994 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T03:59:01.006 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-attr. 2023-10-09T03:59:01.014 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T03:59:01.056 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T03:59:01.322 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T03:59:01.336 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T03:59:01.380 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T03:59:01.383 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T03:59:01.392 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T03:59:01.435 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T03:59:01.683 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T03:59:01.698 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T03:59:01.723 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T03:59:01.737 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T03:59:01.741 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T03:59:01.781 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T03:59:02.052 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T03:59:02.067 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T03:59:02.118 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T03:59:02.133 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T03:59:02.143 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T03:59:02.168 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T03:59:02.645 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T03:59:02.659 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T03:59:02.690 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T03:59:02.703 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T03:59:02.705 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T03:59:02.748 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:03.006 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T03:59:03.020 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T03:59:03.059 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T03:59:03.064 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T03:59:03.074 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T03:59:03.117 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T03:59:03.392 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T03:59:03.407 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T03:59:03.451 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T03:59:03.453 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T03:59:03.467 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T03:59:03.511 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:03.768 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package jq. 2023-10-09T03:59:03.770 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T03:59:03.784 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T03:59:03.785 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T03:59:03.827 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:03.828 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T03:59:04.079 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package socat. 2023-10-09T03:59:04.094 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T03:59:04.137 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T03:59:04.174 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-py. 2023-10-09T03:59:04.188 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T03:59:04.232 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T03:59:04.474 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T03:59:04.489 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T03:59:04.535 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T03:59:04.540 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T03:59:04.548 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T03:59:04.593 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T03:59:04.899 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T03:59:04.911 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T03:59:04.954 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T03:59:04.955 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package ceph-test. 2023-10-09T03:59:04.970 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:05.013 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:05.282 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T03:59:05.296 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T03:59:05.340 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T03:59:05.693 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T03:59:05.707 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T03:59:05.751 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T03:59:06.180 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package radosgw. 2023-10-09T03:59:06.193 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:06.238 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:07.408 INFO:teuthology.orchestra.run.smithi082.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T03:59:07.422 INFO:teuthology.orchestra.run.smithi082.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:07.465 INFO:teuthology.orchestra.run.smithi082.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:07.758 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T03:59:08.054 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T03:59:08.183 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T03:59:08.306 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T03:59:08.502 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:09.110 INFO:teuthology.orchestra.run.smithi082.stdout:Adding system user cephadm....done 2023-10-09T03:59:09.183 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T03:59:09.198 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:09.242 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:09.263 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T03:59:09.616 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T03:59:09.645 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T03:59:09.659 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T03:59:09.718 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T03:59:09.909 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T03:59:10.043 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T03:59:10.097 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T03:59:10.111 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T03:59:10.154 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T03:59:10.370 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T03:59:10.459 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-attr. 2023-10-09T03:59:10.473 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T03:59:10.517 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T03:59:10.755 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T03:59:10.886 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T03:59:10.900 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T03:59:10.943 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T03:59:11.056 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T03:59:11.238 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T03:59:11.252 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T03:59:11.296 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T03:59:11.332 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T03:59:11.600 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T03:59:11.624 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T03:59:11.638 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T03:59:11.682 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T03:59:11.885 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:12.145 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T03:59:12.158 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T03:59:12.194 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T03:59:12.202 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T03:59:12.320 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T03:59:12.514 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T03:59:12.529 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T03:59:12.571 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T03:59:12.604 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T03:59:12.889 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T03:59:12.891 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T03:59:12.906 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T03:59:12.949 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T03:59:13.265 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T03:59:13.277 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T03:59:13.291 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T03:59:13.337 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T03:59:13.549 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T03:59:13.655 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-py. 2023-10-09T03:59:13.670 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T03:59:13.715 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T03:59:13.900 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T03:59:14.033 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T03:59:14.047 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T03:59:14.091 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T03:59:14.176 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:14.302 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T03:59:14.402 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T03:59:14.416 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T03:59:14.460 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T03:59:14.629 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T03:59:14.721 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T03:59:14.737 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T03:59:14.779 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T03:59:14.929 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T03:59:15.082 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T03:59:15.096 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T03:59:15.140 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T03:59:15.222 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T03:59:15.348 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T03:59:15.569 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package radosgw. 2023-10-09T03:59:15.585 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:15.624 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T03:59:15.627 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:16.017 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T03:59:16.386 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T03:59:16.779 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T03:59:16.863 INFO:teuthology.orchestra.run.smithi153.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T03:59:16.878 INFO:teuthology.orchestra.run.smithi153.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T03:59:16.905 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:16.921 INFO:teuthology.orchestra.run.smithi153.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:17.208 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T03:59:17.214 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T03:59:17.494 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T03:59:17.566 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T03:59:17.620 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T03:59:17.692 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T03:59:17.746 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T03:59:17.818 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T03:59:17.955 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:18.102 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T03:59:18.404 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T03:59:18.529 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T03:59:18.592 INFO:teuthology.orchestra.run.smithi153.stdout:Adding system user cephadm....done 2023-10-09T03:59:18.744 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T03:59:18.848 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T03:59:19.096 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T03:59:19.133 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T03:59:19.380 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T03:59:19.426 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T03:59:19.506 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T03:59:19.552 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T03:59:19.828 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T03:59:19.832 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T03:59:20.121 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T03:59:20.210 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T03:59:20.421 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T03:59:20.511 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T03:59:20.787 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T03:59:20.813 INFO:teuthology.orchestra.run.smithi082.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T03:59:21.071 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T03:59:21.074 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T03:59:21.201 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T03:59:21.355 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:21.527 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T03:59:21.656 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T03:59:21.782 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T03:59:21.887 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T03:59:22.013 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T03:59:22.058 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T03:59:22.156 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T03:59:22.351 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T03:59:22.473 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T03:59:22.710 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T03:59:22.758 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:22.884 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T03:59:22.986 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T03:59:23.329 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T03:59:23.622 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:23.748 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T03:59:24.082 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T03:59:24.366 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T03:59:24.387 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:24.512 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:24.638 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:24.676 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T03:59:24.764 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T03:59:24.802 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T03:59:25.078 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T03:59:25.141 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T03:59:25.433 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T03:59:25.463 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T03:59:25.709 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T03:59:25.822 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T03:59:26.061 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:26.199 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T03:59:26.326 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:26.385 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T03:59:26.627 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T03:59:26.970 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T03:59:26.997 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:27.096 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T03:59:27.123 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T03:59:27.447 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T03:59:27.449 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T03:59:28.065 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T03:59:28.374 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T03:59:28.499 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:28.500 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T03:59:28.626 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T03:59:28.752 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:28.818 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T03:59:28.886 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T03:59:29.104 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T03:59:29.178 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T03:59:29.388 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T03:59:29.463 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:29.505 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T03:59:29.591 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:29.781 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T03:59:29.807 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T03:59:29.807 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T03:59:30.074 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T03:59:30.267 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:30.375 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T03:59:30.392 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T03:59:30.685 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T03:59:30.786 INFO:teuthology.orchestra.run.smithi153.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T03:59:30.987 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T03:59:31.038 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T03:59:31.165 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T03:59:31.339 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:31.465 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:31.466 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T03:59:31.591 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:31.717 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T03:59:31.818 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T03:59:31.944 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T03:59:32.051 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T03:59:32.086 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T03:59:32.405 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T03:59:32.429 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:32.555 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:32.697 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:32.831 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T03:59:32.848 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T03:59:33.760 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:34.296 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:34.296 INFO:teuthology.orchestra.run.smithi082.stdout:Adding group ceph....done 2023-10-09T03:59:34.421 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:34.547 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:34.656 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T03:59:34.680 INFO:teuthology.orchestra.run.smithi082.stdout:Adding system user ceph....done 2023-10-09T03:59:34.851 INFO:teuthology.orchestra.run.smithi082.stdout:Setting system user ceph properties....done 2023-10-09T03:59:34.870 INFO:teuthology.orchestra.run.smithi082.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T03:59:35.020 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T03:59:35.033 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T03:59:35.337 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T03:59:35.337 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T03:59:35.622 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T03:59:35.822 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:35.939 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:35.948 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:36.066 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T03:59:36.214 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T03:59:36.214 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T03:59:36.385 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:36.511 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T03:59:36.844 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T03:59:36.883 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:37.243 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T03:59:37.702 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:37.910 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:37.922 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T03:59:37.922 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T03:59:38.036 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T03:59:38.162 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T03:59:38.288 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T03:59:38.381 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:38.589 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T03:59:38.607 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T03:59:38.607 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T03:59:38.882 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:39.007 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:39.091 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:39.230 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T03:59:39.231 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T03:59:39.583 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T03:59:39.583 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T03:59:39.692 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:39.827 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T03:59:40.042 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:40.120 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T03:59:40.169 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:40.311 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:40.445 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T03:59:40.531 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T03:59:40.531 INFO:teuthology.orchestra.run.smithi082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T03:59:40.806 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:40.932 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:40.978 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:41.057 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:41.121 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:41.183 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T03:59:41.249 INFO:teuthology.orchestra.run.smithi082.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:41.393 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T03:59:41.485 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T03:59:41.668 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T03:59:41.871 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:41.998 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:42.307 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T03:59:43.247 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:43.775 INFO:teuthology.orchestra.run.smithi153.stdout:Adding group ceph....done 2023-10-09T03:59:44.152 INFO:teuthology.orchestra.run.smithi153.stdout:Adding system user ceph....done 2023-10-09T03:59:44.321 INFO:teuthology.orchestra.run.smithi153.stdout:Setting system user ceph properties....done 2023-10-09T03:59:44.340 INFO:teuthology.orchestra.run.smithi153.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T03:59:44.494 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T03:59:44.630 INFO:teuthology.orchestra.run.smithi082.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T03:59:44.817 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T03:59:45.081 INFO:teuthology.orchestra.run.smithi082.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T03:59:45.086 DEBUG:teuthology.parallel:result is None 2023-10-09T03:59:45.301 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:45.452 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:45.740 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T03:59:45.740 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T03:59:46.396 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:46.757 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T03:59:47.233 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:47.454 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T03:59:47.454 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T03:59:47.901 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:48.129 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T03:59:48.129 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T03:59:48.594 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:49.093 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T03:59:49.094 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T03:59:49.554 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:49.680 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:49.814 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:50.060 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T03:59:50.060 INFO:teuthology.orchestra.run.smithi153.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T03:59:50.507 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:50.658 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:50.792 INFO:teuthology.orchestra.run.smithi153.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T03:59:50.936 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T03:59:51.237 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T03:59:54.124 INFO:teuthology.orchestra.run.smithi153.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T03:59:54.581 INFO:teuthology.orchestra.run.smithi153.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T03:59:54.584 DEBUG:teuthology.parallel:result is None 2023-10-09T03:59:54.584 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:59:54.584 INFO:teuthology.packaging:ref: None 2023-10-09T03:59:54.584 INFO:teuthology.packaging:tag: None 2023-10-09T03:59:54.584 INFO:teuthology.packaging:branch: pacific 2023-10-09T03:59:54.584 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:59:54.584 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T03:59:54.902 DEBUG:teuthology.orchestra.run.smithi082:> dpkg-query -W -f '${Version}' ceph 2023-10-09T03:59:54.926 INFO:teuthology.orchestra.run.smithi082.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T03:59:54.927 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T03:59:54.927 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T03:59:54.927 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:59:54.927 INFO:teuthology.packaging:ref: None 2023-10-09T03:59:54.927 INFO:teuthology.packaging:tag: None 2023-10-09T03:59:54.927 INFO:teuthology.packaging:branch: pacific 2023-10-09T03:59:54.928 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:59:54.928 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T03:59:55.153 DEBUG:teuthology.orchestra.run.smithi153:> dpkg-query -W -f '${Version}' ceph 2023-10-09T03:59:55.175 INFO:teuthology.orchestra.run.smithi153.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T03:59:55.175 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T03:59:55.175 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T03:59:55.175 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T03:59:55.176 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:59:55.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:59:55.192 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:59:55.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:59:55.230 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T03:59:55.230 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:59:55.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:59:55.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:59:55.304 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:59:55.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:59:55.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:59:55.379 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T03:59:55.379 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:59:55.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:59:55.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:59:55.450 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:59:55.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:59:55.467 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:59:55.521 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T03:59:55.522 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T03:59:55.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:59:55.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:59:55.597 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T03:59:55.597 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:59:55.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:59:55.667 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T03:59:55.745 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:59:55.745 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T03:59:55.745 INFO:tasks.cephadm:Cluster fsid is 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T03:59:55.745 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T03:59:55.746 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.82', 'mon.c': '[v2:172.21.15.82:3301,v1:172.21.15.82:6790]', 'mon.b': '172.21.15.153'} 2023-10-09T03:59:55.746 INFO:tasks.cephadm:First mon is mon.a on smithi082 2023-10-09T03:59:55.746 INFO:tasks.cephadm:First mgr is y 2023-10-09T03:59:55.746 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T03:59:55.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo hostname $(hostname -s) 2023-10-09T03:59:55.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo hostname $(hostname -s) 2023-10-09T03:59:55.780 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T03:59:55.780 DEBUG:teuthology.orchestra.run.smithi082:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:55.963 INFO:teuthology.orchestra.run.smithi082.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 03:59 /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:55.963 DEBUG:teuthology.orchestra.run.smithi153:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:56.117 INFO:teuthology.orchestra.run.smithi153.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 03:59 /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:56.117 DEBUG:teuthology.orchestra.run.smithi082:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:56.127 DEBUG:teuthology.orchestra.run.smithi153:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:56.146 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T03:59:56.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T03:59:56.170 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T03:59:56.434 INFO:teuthology.orchestra.run.smithi082.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T03:59:56.466 INFO:teuthology.orchestra.run.smithi153.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T04:00:41.019 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout: "repo_digests": [ 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:00:41.020 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout: "repo_digests": [ 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T04:00:44.985 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:00:44.986 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:00:45.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo mkdir -p /etc/ceph 2023-10-09T04:00:45.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo mkdir -p /etc/ceph 2023-10-09T04:00:45.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 777 /etc/ceph 2023-10-09T04:00:45.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 777 /etc/ceph 2023-10-09T04:00:45.098 INFO:tasks.cephadm:Writing seed config... 2023-10-09T04:00:45.099 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T04:00:45.100 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T04:00:45.101 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:00:45.101 DEBUG:teuthology.orchestra.run.smithi082:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T04:00:45.131 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T04:00:45.132 DEBUG:teuthology.orchestra.run.smithi082:mon.a> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T04:00:45.175 DEBUG:teuthology.orchestra.run.smithi082:mgr.y> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T04:00:45.218 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T04:00:45.219 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.82 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:00:45.427 INFO:teuthology.orchestra.run.smithi082.stderr:-------------------------------------------------------------------------------- 2023-10-09T04:00:45.427 INFO:teuthology.orchestra.run.smithi082.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '4ea1f4e4-6658-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.82', '--skip-admin-label'] 2023-10-09T04:00:45.427 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying podman|docker is present... 2023-10-09T04:00:45.427 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying lvm2 is present... 2023-10-09T04:00:45.427 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying time synchronization is in place... 2023-10-09T04:00:45.433 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:00:45.438 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.443 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T04:00:45.448 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.453 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: masked 2023-10-09T04:00:45.458 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.463 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T04:00:45.469 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.476 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: enabled 2023-10-09T04:00:45.481 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: active 2023-10-09T04:00:45.482 INFO:teuthology.orchestra.run.smithi082.stderr:Unit ntp.service is enabled and running 2023-10-09T04:00:45.482 INFO:teuthology.orchestra.run.smithi082.stderr:Repeating the final host check... 2023-10-09T04:00:45.482 INFO:teuthology.orchestra.run.smithi082.stderr:docker (/usr/bin/docker) is present 2023-10-09T04:00:45.483 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl is present 2023-10-09T04:00:45.483 INFO:teuthology.orchestra.run.smithi082.stderr:lvcreate is present 2023-10-09T04:00:45.485 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:00:45.491 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.497 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T04:00:45.503 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.509 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: masked 2023-10-09T04:00:45.515 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.520 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T04:00:45.526 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2023-10-09T04:00:45.533 INFO:journalctl@ceph.mgr.y.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:00:45.535 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: enabled 2023-10-09T04:00:45.541 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: active 2023-10-09T04:00:45.542 INFO:teuthology.orchestra.run.smithi082.stderr:Unit ntp.service is enabled and running 2023-10-09T04:00:45.542 INFO:teuthology.orchestra.run.smithi082.stderr:Host looks OK 2023-10-09T04:00:45.542 INFO:teuthology.orchestra.run.smithi082.stderr:Cluster fsid: 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:45.542 INFO:teuthology.orchestra.run.smithi082.stderr:Acquiring lock 140126397476816 on /run/cephadm/4ea1f4e4-6658-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:00:45.543 INFO:teuthology.orchestra.run.smithi082.stderr:Lock 140126397476816 acquired on /run/cephadm/4ea1f4e4-6658-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:00:45.544 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying IP 172.21.15.82 port 3300 ... 2023-10-09T04:00:45.544 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying IP 172.21.15.82 port 6789 ... 2023-10-09T04:00:45.544 INFO:teuthology.orchestra.run.smithi082.stderr:Base mon IP(s) is [172.21.15.82:3300, 172.21.15.82:6789], mon addrv is [v2:172.21.15.82:3300,v1:172.21.15.82:6789] 2023-10-09T04:00:45.546 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-10-09T04:00:45.547 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.82 metric 100 2023-10-09T04:00:45.547 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-09T04:00:45.547 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.82 2023-10-09T04:00:45.547 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.82 metric 100 2023-10-09T04:00:45.551 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T04:00:45.551 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T04:00:45.551 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1791sec pref medium 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d3ff/64 scope link 2023-10-09T04:00:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T04:00:45.556 INFO:teuthology.orchestra.run.smithi082.stderr:Mon IP `172.21.15.82` is in CIDR network `172.21.0.0/20` 2023-10-09T04:00:45.556 INFO:teuthology.orchestra.run.smithi082.stderr:Mon IP `172.21.15.82` is in CIDR network `172.21.0.0/20` 2023-10-09T04:00:45.557 INFO:teuthology.orchestra.run.smithi082.stderr:Mon IP `172.21.15.82` is in CIDR network `172.21.15.254/32` 2023-10-09T04:00:45.557 INFO:teuthology.orchestra.run.smithi082.stderr:Mon IP `172.21.15.82` is in CIDR network `172.21.15.254/32` 2023-10-09T04:00:45.557 INFO:teuthology.orchestra.run.smithi082.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-10-09T04:00:45.558 INFO:teuthology.orchestra.run.smithi082.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T04:00:45.558 INFO:teuthology.orchestra.run.smithi082.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T04:00:45.560 INFO:journalctl@ceph.mon.a.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:00:45.726 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-10-09T04:00:45.748 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/docker: Digest: sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3 2023-10-09T04:00:45.748 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T04:00:45.764 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T04:00:46.699 INFO:teuthology.orchestra.run.smithi082.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T04:00:47.264 INFO:teuthology.orchestra.run.smithi082.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T04:00:47.264 INFO:teuthology.orchestra.run.smithi082.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T04:00:47.971 INFO:teuthology.orchestra.run.smithi082.stderr:stat: 167 167 2023-10-09T04:00:48.500 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial keys... 2023-10-09T04:00:49.239 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQDxeiNl2QMrDhAABru1vRVFEAURRl/mQWlWww== 2023-10-09T04:00:50.546 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQDyeiNlAv96IBAA9KsjoTqZtLNuDjHGoqcpEw== 2023-10-09T04:00:51.847 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQDzeiNl/DxxMhAAqAR9skU5H66meco/OAsRHA== 2023-10-09T04:00:52.400 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial monmap... 2023-10-09T04:00:53.176 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:00:53.176 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:53.176 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:00:53.735 INFO:teuthology.orchestra.run.smithi082.stderr:monmaptool for a [v2:172.21.15.82:3300,v1:172.21.15.82:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:00:53.735 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: set fsid to 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:53.735 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:00:53.735 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:00:53.735 INFO:teuthology.orchestra.run.smithi082.stderr:Creating mon... 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.483+0000 7f7288f76880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.483+0000 7f7288f76880 1 imported monmap: 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T04:00:53.174390+0000 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: created 2023-10-09T04:00:53.174390+0000 2023-10-09T04:00:54.491 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T04:00:54.492 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T04:00:54.492 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] mon.a 2023-10-09T04:00:54.492 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.492 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.483+0000 7f7288f76880 0 /usr/bin/ceph-mon: set fsid to 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:54.494 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T04:00:54.494 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: DB SUMMARY 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T04:00:54.495 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.env: 0x558c035c1080 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.info_log: 0x558c04cd9aa0 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:00:54.496 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:00:54.497 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:00:54.497 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:00:54.518 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:00:54.518 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.db_log_dir: 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:00:54.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.write_buffer_manager: 0x558c04ce1890 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:00:54.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.row_cache: None 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.wal_filter: None 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:00:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:00:54.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:00:54.523 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Compression algorithms supported: 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kZSTD supported: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:00:54.524 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.merge_operator: 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:00:54.525 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558c04befd20) 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T04:00:54.526 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_cache: 0x558c04c26d10 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T04:00:54.527 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.528 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.num_levels: 7 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:00:54.529 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:00:54.530 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:00:54.531 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:00:54.532 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:00:54.533 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.487+0000 7f7288f76880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.491+0000 7f7288f76880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.534 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.491+0000 7f7288f76880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.491+0000 7f7288f76880 4 rocksdb: DB pointer 0x558c04cef800 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.491+0000 7f7271dfe700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.491+0000 7f7271dfe700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:00:54.535 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:00:54.537 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.495+0000 7f7288f76880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T04:00:54.539 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.495+0000 7f7288f76880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T04:00:54.540 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-mon: debug 2023-10-09T04:00:54.495+0000 7f7288f76880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T04:00:55.074 INFO:teuthology.orchestra.run.smithi082.stderr:create mon.a on 2023-10-09T04:00:55.104 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T04:00:55.313 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-09T04:00:55.527 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:00:55.527 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:00:55.953 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to reset failed state of unit ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T04:00:55.962 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target.wants/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:00:56.166 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld does not appear to be present 2023-10-09T04:00:56.166 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T04:00:56.167 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mon to start... 2023-10-09T04:00:56.167 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mon... 2023-10-09T04:00:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:57 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.347218+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: cluster: 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: id: 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: services: 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.250208s) 2023-10-09T04:00:57.604 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T04:00:57.605 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: data: 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: pgs: 2023-10-09T04:00:57.606 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:58.148 INFO:teuthology.orchestra.run.smithi082.stderr:mon is available 2023-10-09T04:00:58.148 INFO:teuthology.orchestra.run.smithi082.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T04:00:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.351535+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:00:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.351714+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0]} removed_ranks: {} 2023-10-09T04:00:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.354920+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T04:00:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.359116+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:00:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: cluster 2023-10-09T04:00:57.359682+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T04:00:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:58 smithi082 bash[26926]: audit 2023-10-09T04:00:57.601666+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.82:0/2457537508' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T04:00:59.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:59.208 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [global] 2023-10-09T04:00:59.208 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: fsid = 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:59.209 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.82:3300,v1:172.21.15.82:6789] 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [mgr] 2023-10-09T04:00:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T04:00:59.211 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:00:59.211 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [osd] 2023-10-09T04:00:59.211 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T04:00:59.211 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T04:00:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:59 smithi082 bash[26926]: audit 2023-10-09T04:00:59.199243+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.82:0/2878744869' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:00:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:00:59 smithi082 bash[26926]: audit 2023-10-09T04:00:59.205185+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.82:0/2878744869' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T04:00:59.819 INFO:teuthology.orchestra.run.smithi082.stderr:Generating new minimal ceph.conf... 2023-10-09T04:01:01.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:00 smithi082 bash[26926]: audit 2023-10-09T04:01:00.843724+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.82:0/3236649125' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:01.407 INFO:teuthology.orchestra.run.smithi082.stderr:Restarting the monitor... 2023-10-09T04:01:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:01 smithi082 systemd[1]: Stopping Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:01:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:01 smithi082 bash[27324]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon.a 2023-10-09T04:01:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:01 smithi082 bash[26926]: debug 2023-10-09T04:01:01.587+0000 7fc2d7cf9700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:01:01.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:01 smithi082 bash[26926]: debug 2023-10-09T04:01:01.587+0000 7fc2d7cf9700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T04:01:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:02 smithi082 bash[27361]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:01:02.350 INFO:teuthology.orchestra.run.smithi082.stderr:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-10-09T04:01:02.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:02 smithi082 bash[27425]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:01:02.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:02 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:01:02.410 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:02 smithi082 systemd[1]: Stopped Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:01:02.410 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:02 smithi082 systemd[1]: Started Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.419+0000 7fdd42ef5880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.419+0000 7fdd42ef5880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.419+0000 7fdd42ef5880 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 0 load: jerasure load: lrc load: isa 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T04:01:03.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: DB SUMMARY 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86707 ; 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.env: 0x55eddfbc1080 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.info_log: 0x55ede185ba80 2023-10-09T04:01:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:01:03.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:01:03.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.db_log_dir: 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:01:03.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.write_buffer_manager: 0x55ede2558330 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:01:03.737 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.row_cache: None 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.wal_filter: None 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:01:03.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:01:03.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Compression algorithms supported: 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kZSTD supported: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:01:03.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.merge_operator: 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:01:03.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ede1809d18) 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cache_index_and_filter_blocks: 1 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: pin_top_level_index_and_filter: 1 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: index_type: 0 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: data_block_index_type: 0 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: index_shortening: 1 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: hash_index_allow_collision: 1 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: checksum: 1 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: no_block_cache: 0 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_cache: 0x55ede1840f10 2023-10-09T04:01:03.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_cache_name: BinnedLRUCache 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_cache_options: 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: capacity : 536870912 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: num_shard_bits : 4 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: strict_capacity_limit : 0 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: high_pri_pool_ratio: 0.000 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_cache_compressed: (nil) 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: persistent_cache: (nil) 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_size: 4096 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_size_deviation: 10 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_restart_interval: 16 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: index_block_restart_interval: 1 2023-10-09T04:01:03.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: metadata_block_size: 4096 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: partition_filters: 0 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: use_delta_encoding: 1 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: whole_key_filtering: 1 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: verify_compression: 0 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: read_amp_bytes_per_bit: 0 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: format_version: 2 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: enable_index_compression: 1 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: block_align: 0 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:01:03.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.num_levels: 7 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:01:03.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:01:03.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:01:03.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:01:03.748 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:01:03.748 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:01:03.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T04:01:03.750 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824063432807, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.427+0000 7fdd42ef5880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824063433940, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83307, "table_properties": {"data_size": 81580, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75901, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696824063, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824063434426, "job": 1, "event": "recovery_finished"} 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 4 rocksdb: DB pointer 0x55ede1909800 2023-10-09T04:01:03.751 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd29cfc700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd29cfc700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** DB Stats ** 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** Compaction Stats [default] ** 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:01:03.752 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** Compaction Stats [default] ** 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:01:03.753 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Cumulative compaction: 0.00 GB write, 21.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Interval compaction: 0.00 GB write, 21.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** Compaction Stats [default] ** 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:01:03.754 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** Compaction Stats [default] ** 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 114.5 0.00 0.00 1 0.001 0 0 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Cumulative compaction: 0.00 GB write, 21.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:01:03.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.431+0000 7fdd42ef5880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] at bind addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 1 mon.a@-1(???) e1 preinit fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).mds e1 new map 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: e1 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: legacy client fscid: -1 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: No filesystems configured 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:01:03.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: audit 2023-10-09T04:01:00.843724+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.82:0/3236649125' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: debug 2023-10-09T04:01:03.435+0000 7fdd42ef5880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cluster 2023-10-09T04:01:03.442870+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cluster 2023-10-09T04:01:03.442974+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0]} removed_ranks: {} 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cluster 2023-10-09T04:01:03.443579+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cluster 2023-10-09T04:01:03.443642+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:01:03.757 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:03 smithi082 bash[27462]: cluster 2023-10-09T04:01:03.444167+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T04:01:04.393 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T04:01:04.393 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:01:04.393 INFO:teuthology.orchestra.run.smithi082.stderr:Creating mgr... 2023-10-09T04:01:04.394 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying port 9283 ... 2023-10-09T04:01:04.621 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to reset failed state of unit ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T04:01:04.630 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7.target.wants/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:01:04.840 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld does not appear to be present 2023-10-09T04:01:04.841 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T04:01:04.841 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld does not appear to be present 2023-10-09T04:01:04.841 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T04:01:04.842 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr to start... 2023-10-09T04:01:04.842 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr... 2023-10-09T04:01:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:04 smithi082 bash[27462]: audit 2023-10-09T04:01:03.703583+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.82:0/2067490467' entity='client.admin' 2023-10-09T04:01:06.366 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "4ea1f4e4-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:06.367 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "a" 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T04:01:06.368 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T04:01:06.369 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": false, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T04:01:06.370 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2023-10-09T04:00:57.353168+0000", 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T04:01:06.371 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:06.552 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:06 smithi082 bash[27740]: debug 2023-10-09T04:01:06.355+0000 7f1dc3558680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:06.553 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:06 smithi082 bash[27462]: audit 2023-10-09T04:01:06.357613+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.82:0/3763993782' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:01:06.957 INFO:teuthology.orchestra.run.smithi082.stderr:mgr not available, waiting (1/15)... 2023-10-09T04:01:07.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:06 smithi082 bash[27740]: debug 2023-10-09T04:01:06.835+0000 7f1dc3558680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:01:08.112 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:07 smithi082 bash[27740]: debug 2023-10-09T04:01:07.755+0000 7f1dc3558680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:08.113 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:07 smithi082 bash[27740]: debug 2023-10-09T04:01:07.855+0000 7f1dc3558680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:01:08.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:08 smithi082 bash[27740]: debug 2023-10-09T04:01:08.107+0000 7f1dc3558680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:01:09.546 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:09 smithi082 bash[27740]: debug 2023-10-09T04:01:09.271+0000 7f1dc3558680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:09.546 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:09 smithi082 bash[27740]: debug 2023-10-09T04:01:09.439+0000 7f1dc3558680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:09.888 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:09 smithi082 bash[27740]: debug 2023-10-09T04:01:09.543+0000 7f1dc3558680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:09.888 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:09 smithi082 bash[27740]: debug 2023-10-09T04:01:09.659+0000 7f1dc3558680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:10.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:10.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "4ea1f4e4-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T04:01:10.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "a" 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T04:01:10.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T04:01:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T04:01:10.045 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T04:01:10.045 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T04:01:10.046 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": false, 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2023-10-09T04:01:10.047 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2023-10-09T04:00:57.353168+0000", 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T04:01:10.048 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:10 smithi082 bash[27462]: audit 2023-10-09T04:01:10.034538+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.82:0/398307743' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:01:10.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:09 smithi082 bash[27740]: debug 2023-10-09T04:01:09.883+0000 7f1dc3558680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:01:10.646 INFO:teuthology.orchestra.run.smithi082.stderr:mgr not available, waiting (2/15)... 2023-10-09T04:01:10.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:10 smithi082 bash[27740]: debug 2023-10-09T04:01:10.303+0000 7f1dc3558680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:01:10.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:10 smithi082 bash[27740]: debug 2023-10-09T04:01:10.431+0000 7f1dc3558680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:11.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:11 smithi082 bash[27740]: debug 2023-10-09T04:01:11.335+0000 7f1dc3558680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:01:11.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:11 smithi082 bash[27740]: debug 2023-10-09T04:01:11.435+0000 7f1dc3558680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:01:11.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:11 smithi082 bash[27740]: debug 2023-10-09T04:01:11.551+0000 7f1dc3558680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:01:12.062 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:11 smithi082 bash[27740]: debug 2023-10-09T04:01:11.771+0000 7f1dc3558680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:01:12.396 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27740]: debug 2023-10-09T04:01:12.059+0000 7f1dc3558680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:12.396 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27740]: debug 2023-10-09T04:01:12.223+0000 7f1dc3558680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:01:12.679 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27740]: debug 2023-10-09T04:01:12.391+0000 7f1dc3558680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:12.679 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27740]: debug 2023-10-09T04:01:12.607+0000 7f1dc3558680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:12.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27740]: debug 2023-10-09T04:01:12.707+0000 7f1dc3558680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:01:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: cluster 2023-10-09T04:01:12.731380+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: cluster 2023-10-09T04:01:12.737495+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00633531s) 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.739389+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.739743+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.740028+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.740331+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.740918+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: cluster 2023-10-09T04:01:12.764346+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-10-09T04:01:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:12 smithi082 bash[27462]: audit 2023-10-09T04:01:12.775223+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "4ea1f4e4-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T04:01:14.197 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:14.198 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T04:01:14.199 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "a" 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T04:01:14.200 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T04:01:14.201 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T04:01:14.201 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T04:01:14.204 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T04:01:14.204 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T04:01:14.204 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T04:01:14.205 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:14.206 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2023-10-09T04:00:57.353168+0000", 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T04:01:14.207 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:14.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:13 smithi082 bash[27462]: audit 2023-10-09T04:01:12.783084+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:14.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:13 smithi082 bash[27462]: audit 2023-10-09T04:01:12.787054+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:14.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:13 smithi082 bash[27462]: audit 2023-10-09T04:01:12.791097+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' 2023-10-09T04:01:14.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:13 smithi082 bash[27462]: audit 2023-10-09T04:01:12.793607+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.82:0/154164404' entity='mgr.y' 2023-10-09T04:01:14.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:13 smithi082 bash[27462]: cluster 2023-10-09T04:01:13.741630+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01048s) 2023-10-09T04:01:14.785 INFO:teuthology.orchestra.run.smithi082.stderr:mgr is available 2023-10-09T04:01:15.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:14 smithi082 bash[27462]: audit 2023-10-09T04:01:14.193821+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.82:0/1918202562' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:01:15.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:15 smithi082 bash[27462]: cluster 2023-10-09T04:01:14.789693+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [global] 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: fsid = 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:16.145 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [mgr] 2023-10-09T04:01:16.146 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T04:01:16.146 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2023-10-09T04:01:16.146 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [osd] 2023-10-09T04:01:16.146 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T04:01:16.146 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T04:01:16.733 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling cephadm module... 2023-10-09T04:01:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:16 smithi082 bash[27462]: audit 2023-10-09T04:01:16.141199+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.82:0/1099343875' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:01:18.450 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27462]: audit 2023-10-09T04:01:18.133304+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.82:0/1837196693' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T04:01:18.450 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27740]: ignoring --setuser ceph since I am not root 2023-10-09T04:01:18.451 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27740]: ignoring --setgroup ceph since I am not root 2023-10-09T04:01:18.451 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27740]: debug 2023-10-09T04:01:18.383+0000 7f1b95cb1680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:01:18.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27740]: debug 2023-10-09T04:01:18.491+0000 7f1b95cb1680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:19.193 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:18 smithi082 bash[27740]: debug 2023-10-09T04:01:18.939+0000 7f1b95cb1680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:01:19.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:19 smithi082 bash[27462]: audit 2023-10-09T04:01:18.188444+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.82:0/1837196693' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T04:01:19.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:19 smithi082 bash[27462]: cluster 2023-10-09T04:01:18.188486+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 5s) 2023-10-09T04:01:19.975 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:19 smithi082 bash[27740]: debug 2023-10-09T04:01:19.851+0000 7f1b95cb1680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:20.227 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:19 smithi082 bash[27740]: debug 2023-10-09T04:01:19.971+0000 7f1b95cb1680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T04:01:20.293 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:20 smithi082 bash[27462]: audit 2023-10-09T04:01:20.290722+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.82:0/1944114047' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:01:20.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:20 smithi082 bash[27740]: debug 2023-10-09T04:01:20.223+0000 7f1b95cb1680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:01:20.973 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for the mgr to restart... 2023-10-09T04:01:20.974 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr epoch 5... 2023-10-09T04:01:21.621 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:21 smithi082 bash[27740]: debug 2023-10-09T04:01:21.359+0000 7f1b95cb1680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:21.621 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:21 smithi082 bash[27740]: debug 2023-10-09T04:01:21.519+0000 7f1b95cb1680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:21.621 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:21 smithi082 bash[27740]: debug 2023-10-09T04:01:21.615+0000 7f1b95cb1680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:21.966 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:21 smithi082 bash[27740]: debug 2023-10-09T04:01:21.735+0000 7f1b95cb1680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:22.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:21 smithi082 bash[27740]: debug 2023-10-09T04:01:21.959+0000 7f1b95cb1680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:01:22.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:22 smithi082 bash[27740]: debug 2023-10-09T04:01:22.323+0000 7f1b95cb1680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:01:22.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:22 smithi082 bash[27740]: debug 2023-10-09T04:01:22.443+0000 7f1b95cb1680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:23.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:23 smithi082 bash[27740]: debug 2023-10-09T04:01:23.299+0000 7f1b95cb1680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:01:23.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:23 smithi082 bash[27740]: debug 2023-10-09T04:01:23.407+0000 7f1b95cb1680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:01:23.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:23 smithi082 bash[27740]: debug 2023-10-09T04:01:23.527+0000 7f1b95cb1680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:01:24.048 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:23 smithi082 bash[27740]: debug 2023-10-09T04:01:23.755+0000 7f1b95cb1680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:01:24.381 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27740]: debug 2023-10-09T04:01:24.043+0000 7f1b95cb1680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:24.381 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27740]: debug 2023-10-09T04:01:24.207+0000 7f1b95cb1680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:01:24.699 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27740]: debug 2023-10-09T04:01:24.375+0000 7f1b95cb1680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:24.699 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27740]: debug 2023-10-09T04:01:24.587+0000 7f1b95cb1680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:24.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27740]: debug 2023-10-09T04:01:24.695+0000 7f1b95cb1680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:01:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: cluster 2023-10-09T04:01:24.719135+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: cluster 2023-10-09T04:01:24.719553+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: cluster 2023-10-09T04:01:24.725601+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: cluster 2023-10-09T04:01:24.725861+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00645562s) 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.726864+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.727835+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.728645+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.728813+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.729003+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: cluster 2023-10-09T04:01:24.732114+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-09T04:01:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:24 smithi082 bash[27462]: audit 2023-10-09T04:01:24.750129+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:25.745 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:25.745 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T04:01:25.745 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T04:01:25.745 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: cephadm 2023-10-09T04:01:24.747175+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.878851+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.881584+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.884574+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.887830+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.908728+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: audit 2023-10-09T04:01:24.913521+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:25 smithi082 bash[27462]: cluster 2023-10-09T04:01:25.730530+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01112s) 2023-10-09T04:01:26.396 INFO:teuthology.orchestra.run.smithi082.stderr:mgr epoch 5 is available 2023-10-09T04:01:26.396 INFO:teuthology.orchestra.run.smithi082.stderr:Setting orchestrator backend to cephadm... 2023-10-09T04:01:27.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:26 smithi082 bash[27462]: audit 2023-10-09T04:01:25.729392+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:01:27.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:26 smithi082 bash[27462]: audit 2023-10-09T04:01:25.743382+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:01:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:27 smithi082 bash[27462]: cluster 2023-10-09T04:01:26.880887+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T04:01:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:27 smithi082 bash[27462]: audit 2023-10-09T04:01:27.710798+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:28.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:27 smithi082 bash[27462]: audit 2023-10-09T04:01:27.769231+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:29.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:28 smithi082 bash[27462]: audit 2023-10-09T04:01:27.701228+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:29.596 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: value unchanged 2023-10-09T04:01:30.181 INFO:teuthology.orchestra.run.smithi082.stderr:Generating ssh key... 2023-10-09T04:01:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:30 smithi082 bash[27462]: audit 2023-10-09T04:01:29.593753+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:31.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: Generating public/private rsa key pair. 2023-10-09T04:01:31.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: Your identification has been saved in /tmp/tmpzuj5f4pl/key. 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: Your public key has been saved in /tmp/tmpzuj5f4pl/key.pub. 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: The key fingerprint is: 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: SHA256:IGvXo7+IB0UieEUrONm/LsEosHVowF5aSeYzQPg9Ef8 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: The key's randomart image is: 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: +---[RSA 3072]----+ 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |++.==. | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |+=*+oo. | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |=+*Bo+o | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |.+=o*o.+ | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |.* .+o. E | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |+ o..o . . | 2023-10-09T04:01:31.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: |. .... | 2023-10-09T04:01:31.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: | .. ..o | 2023-10-09T04:01:31.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: | .o.. o. | 2023-10-09T04:01:31.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:31 smithi082 bash[27740]: +----[SHA256]-----+ 2023-10-09T04:01:32.789 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:32 smithi082 bash[27462]: audit 2023-10-09T04:01:31.411747+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:32.789 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:32 smithi082 bash[27462]: cephadm 2023-10-09T04:01:31.412200+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-09T04:01:32.789 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:32 smithi082 bash[27462]: audit 2023-10-09T04:01:31.489799+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:32.789 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:32 smithi082 bash[27462]: audit 2023-10-09T04:01:31.493155+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:33.335 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDhXBRxXVM1oEqmX7V0b34nPDO9KZAeqUJyJSErCC6+Kviu1preO52pPZ8NZUFCZONkNzPMquVFCGNcvu83+LLsrou8Mxq+WWiRPncu3nw1wpqP5JUNlFdtC8/v9QUBvK7Gx9f6BmLmI14slp2aBWqnrpPldNeSKFkjscZPZdkSd9FypMJNl5m/235Rz2au7WFoGjoxV66HTtDRtpjyKcpAqA0DvzOBmB+FGaKmhVjMe0oEhZMugdzjveceV9P6mGInBy7SfIZEqNa2eQl2PWPDCiazYNjotaXU7dxuOqai1GEsB5tayb9VNLHv0obsuYyE9oWUn9Alu3W/s1VI5waysbA2GUxghYDOSUbjYKoEmVFF6GOXqPZhu/M7aObulf6bYzlHcL4iOfaNJDLNToqiY5zGrHOupa1WcP5jKnRDPdgnNBlOAfltljOn9AzeoM4Y2Dq+qXzuJFA2sOpNgd71LxWsLMd6rowcXAggwrD/DM86pYXpNiRCDZfp0cDVSs= ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:33.967 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T04:01:33.967 INFO:teuthology.orchestra.run.smithi082.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T04:01:33.968 INFO:teuthology.orchestra.run.smithi082.stderr:Adding host smithi082... 2023-10-09T04:01:34.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:34 smithi082 bash[27462]: audit 2023-10-09T04:01:33.332955+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:35 smithi082 bash[27462]: audit 2023-10-09T04:01:35.237191+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi082", "addr": "172.21.15.82", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:35.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:35 smithi082 bash[27740]: Warning: Permanently added '172.21.15.82' (ECDSA) to the list of known hosts. 2023-10-09T04:01:36.063 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Added host 'smithi082' with addr '172.21.15.82' 2023-10-09T04:01:36.920 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying unmanaged mon service... 2023-10-09T04:01:37.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:37 smithi082 bash[27462]: audit 2023-10-09T04:01:36.060693+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:37.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:37 smithi082 bash[27462]: cephadm 2023-10-09T04:01:36.061230+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi082 2023-10-09T04:01:37.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:37 smithi082 bash[27462]: audit 2023-10-09T04:01:36.109457+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:38.780 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T04:01:39.673 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying unmanaged mgr service... 2023-10-09T04:01:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:39 smithi082 bash[27462]: audit 2023-10-09T04:01:38.773873+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:39 smithi082 bash[27462]: cephadm 2023-10-09T04:01:38.775274+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T04:01:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:39 smithi082 bash[27462]: audit 2023-10-09T04:01:38.779207+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:39 smithi082 bash[27462]: audit 2023-10-09T04:01:39.367734+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:39 smithi082 bash[27462]: audit 2023-10-09T04:01:39.614474+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:41.098 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T04:01:42.430 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:42 smithi082 bash[27462]: audit 2023-10-09T04:01:41.089602+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:42.430 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:42 smithi082 bash[27462]: cephadm 2023-10-09T04:01:41.091705+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T04:01:42.431 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:42 smithi082 bash[27462]: audit 2023-10-09T04:01:41.096391+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:44.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:44 smithi082 bash[27462]: audit 2023-10-09T04:01:43.057388+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.82:0/3105409341' entity='client.admin' 2023-10-09T04:01:45.689 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling the dashboard module... 2023-10-09T04:01:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:45 smithi082 bash[27462]: cluster 2023-10-09T04:01:44.730422+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:01:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:45 smithi082 bash[27462]: audit 2023-10-09T04:01:44.816811+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.82:0/320574113' entity='mgr.y' 2023-10-09T04:01:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:45 smithi082 bash[27462]: audit 2023-10-09T04:01:45.072624+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.82:0/2529804139' entity='client.admin' 2023-10-09T04:01:48.101 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:47 smithi082 bash[27462]: cluster 2023-10-09T04:01:46.730816+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:01:48.101 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:47 smithi082 bash[27462]: audit 2023-10-09T04:01:47.071684+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.82:0/3474825211' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T04:01:48.102 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:47 smithi082 bash[27740]: ignoring --setuser ceph since I am not root 2023-10-09T04:01:48.102 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:47 smithi082 bash[27740]: ignoring --setgroup ceph since I am not root 2023-10-09T04:01:48.102 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:48 smithi082 bash[27740]: debug 2023-10-09T04:01:48.034+0000 7fa71b225680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:01:48.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:48 smithi082 bash[27740]: debug 2023-10-09T04:01:48.130+0000 7fa71b225680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:48.824 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:48 smithi082 bash[27740]: debug 2023-10-09T04:01:48.562+0000 7fa71b225680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:01:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:48 smithi082 bash[27462]: audit 2023-10-09T04:01:47.827442+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.82:0/3474825211' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T04:01:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:48 smithi082 bash[27462]: cluster 2023-10-09T04:01:47.827589+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 23s) 2023-10-09T04:01:49.623 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:49 smithi082 bash[27740]: debug 2023-10-09T04:01:49.514+0000 7fa71b225680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:49.877 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:49 smithi082 bash[27740]: debug 2023-10-09T04:01:49.618+0000 7fa71b225680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T04:01:50.022 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:50.226 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:50 smithi082 bash[27462]: audit 2023-10-09T04:01:50.020196+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.82:0/3077127621' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:01:50.226 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:49 smithi082 bash[27740]: debug 2023-10-09T04:01:49.870+0000 7fa71b225680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:01:50.662 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for the mgr to restart... 2023-10-09T04:01:50.663 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr epoch 9... 2023-10-09T04:01:51.317 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:51 smithi082 bash[27740]: debug 2023-10-09T04:01:51.034+0000 7fa71b225680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:51.318 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:51 smithi082 bash[27740]: debug 2023-10-09T04:01:51.210+0000 7fa71b225680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:51.662 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:51 smithi082 bash[27740]: debug 2023-10-09T04:01:51.310+0000 7fa71b225680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:51.663 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:51 smithi082 bash[27740]: debug 2023-10-09T04:01:51.430+0000 7fa71b225680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:52.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:51 smithi082 bash[27740]: debug 2023-10-09T04:01:51.658+0000 7fa71b225680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:01:52.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:52 smithi082 bash[27740]: debug 2023-10-09T04:01:52.030+0000 7fa71b225680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:01:52.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:52 smithi082 bash[27740]: debug 2023-10-09T04:01:52.158+0000 7fa71b225680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:53.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.030+0000 7fa71b225680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:01:53.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.142+0000 7fa71b225680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:01:53.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.262+0000 7fa71b225680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:01:53.806 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.486+0000 7fa71b225680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:01:54.142 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.802+0000 7fa71b225680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:54.143 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:53 smithi082 bash[27740]: debug 2023-10-09T04:01:53.966+0000 7fa71b225680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:01:54.470 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27740]: debug 2023-10-09T04:01:54.138+0000 7fa71b225680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:54.470 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27740]: debug 2023-10-09T04:01:54.358+0000 7fa71b225680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:54.724 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27740]: debug 2023-10-09T04:01:54.466+0000 7fa71b225680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:01:54.724 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: cluster 2023-10-09T04:01:54.491408+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: cluster 2023-10-09T04:01:54.491847+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: cluster 2023-10-09T04:01:54.500065+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: cluster 2023-10-09T04:01:54.500243+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00857237s) 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: audit 2023-10-09T04:01:54.501374+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: audit 2023-10-09T04:01:54.502893+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: audit 2023-10-09T04:01:54.503703+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: audit 2023-10-09T04:01:54.503855+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: audit 2023-10-09T04:01:54.503988+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:54.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27462]: cluster 2023-10-09T04:01:54.525400+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-10-09T04:01:54.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:01:54 smithi082 bash[27740]: Warning: Permanently added '172.21.15.82' (ECDSA) to the list of known hosts. 2023-10-09T04:01:55.519 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2023-10-09T04:01:55.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T04:01:55.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T04:01:55.520 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2023-10-09T04:01:55.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:55 smithi082 bash[27462]: audit 2023-10-09T04:01:54.654477+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:55.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:55 smithi082 bash[27462]: audit 2023-10-09T04:01:54.659106+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:55.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:55 smithi082 bash[27462]: audit 2023-10-09T04:01:54.696011+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:55.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:55 smithi082 bash[27462]: audit 2023-10-09T04:01:54.701333+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:55.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:55 smithi082 bash[27462]: cluster 2023-10-09T04:01:55.503559+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.01189s) 2023-10-09T04:01:56.175 INFO:teuthology.orchestra.run.smithi082.stderr:mgr epoch 9 is available 2023-10-09T04:01:56.175 INFO:teuthology.orchestra.run.smithi082.stderr:Generating a dashboard self-signed certificate... 2023-10-09T04:01:56.869 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:56 smithi082 bash[27462]: audit 2023-10-09T04:01:55.502413+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:01:56.869 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:56 smithi082 bash[27462]: audit 2023-10-09T04:01:55.518339+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:01:57.611 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T04:01:57.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:57 smithi082 bash[27462]: cluster 2023-10-09T04:01:56.554668+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T04:01:58.257 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial admin user... 2023-10-09T04:01:58.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:58 smithi082 bash[27462]: audit 2023-10-09T04:01:57.467562+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:58.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:58 smithi082 bash[27462]: audit 2023-10-09T04:01:57.604039+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:01:58.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:01:58 smithi082 bash[27462]: audit 2023-10-09T04:01:57.610066+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:01:59.787 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$SQCfUztSSMt1KjphF7jUlui7Qz2IuTN6agSEHzMLmJCeX7hn035FO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696824119, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T04:02:00.463 INFO:teuthology.orchestra.run.smithi082.stderr:Fetching dashboard port number... 2023-10-09T04:02:01.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:00 smithi082 bash[27462]: audit 2023-10-09T04:01:59.533022+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:01.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:00 smithi082 bash[27462]: audit 2023-10-09T04:01:59.782205+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:01.762 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 8443 2023-10-09T04:02:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:01 smithi082 bash[27462]: cluster 2023-10-09T04:02:00.784884+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T04:02:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:01 smithi082 bash[27462]: audit 2023-10-09T04:02:01.280731+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:01 smithi082 bash[27462]: audit 2023-10-09T04:02:01.761155+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.82:0/3780718781' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T04:02:02.402 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld does not appear to be present 2023-10-09T04:02:02.402 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr:Ceph Dashboard is now available at: 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr: URL: https://smithi082.front.sepia.ceph.com:8443/ 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr: User: admin 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr: Password: ztioayks77 2023-10-09T04:02:02.406 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:02.407 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling autotune for osd_memory_target 2023-10-09T04:02:05.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T04:02:06.482 INFO:teuthology.orchestra.run.smithi082.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T04:02:06.482 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.482 INFO:teuthology.orchestra.run.smithi082.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:06.482 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.482 INFO:teuthology.orchestra.run.smithi082.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.483 INFO:teuthology.orchestra.run.smithi082.stderr: ceph telemetry on 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr:For more information see: 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr: 2023-10-09T04:02:06.484 INFO:teuthology.orchestra.run.smithi082.stderr:Bootstrap complete. 2023-10-09T04:02:06.525 INFO:tasks.cephadm:Fetching config... 2023-10-09T04:02:06.525 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:06.525 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T04:02:06.535 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T04:02:06.535 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:06.535 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T04:02:06.581 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T04:02:06.581 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:06.581 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T04:02:06.639 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T04:02:06.639 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:06.639 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T04:02:06.686 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T04:02:06.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDhXBRxXVM1oEqmX7V0b34nPDO9KZAeqUJyJSErCC6+Kviu1preO52pPZ8NZUFCZONkNzPMquVFCGNcvu83+LLsrou8Mxq+WWiRPncu3nw1wpqP5JUNlFdtC8/v9QUBvK7Gx9f6BmLmI14slp2aBWqnrpPldNeSKFkjscZPZdkSd9FypMJNl5m/235Rz2au7WFoGjoxV66HTtDRtpjyKcpAqA0DvzOBmB+FGaKmhVjMe0oEhZMugdzjveceV9P6mGInBy7SfIZEqNa2eQl2PWPDCiazYNjotaXU7dxuOqai1GEsB5tayb9VNLHv0obsuYyE9oWUn9Alu3W/s1VI5waysbA2GUxghYDOSUbjYKoEmVFF6GOXqPZhu/M7aObulf6bYzlHcL4iOfaNJDLNToqiY5zGrHOupa1WcP5jKnRDPdgnNBlOAfltljOn9AzeoM4Y2Dq+qXzuJFA2sOpNgd71LxWsLMd6rowcXAggwrD/DM86pYXpNiRCDZfp0cDVSs= ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:02:06.750 INFO:teuthology.orchestra.run.smithi082.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDhXBRxXVM1oEqmX7V0b34nPDO9KZAeqUJyJSErCC6+Kviu1preO52pPZ8NZUFCZONkNzPMquVFCGNcvu83+LLsrou8Mxq+WWiRPncu3nw1wpqP5JUNlFdtC8/v9QUBvK7Gx9f6BmLmI14slp2aBWqnrpPldNeSKFkjscZPZdkSd9FypMJNl5m/235Rz2au7WFoGjoxV66HTtDRtpjyKcpAqA0DvzOBmB+FGaKmhVjMe0oEhZMugdzjveceV9P6mGInBy7SfIZEqNa2eQl2PWPDCiazYNjotaXU7dxuOqai1GEsB5tayb9VNLHv0obsuYyE9oWUn9Alu3W/s1VI5waysbA2GUxghYDOSUbjYKoEmVFF6GOXqPZhu/M7aObulf6bYzlHcL4iOfaNJDLNToqiY5zGrHOupa1WcP5jKnRDPdgnNBlOAfltljOn9AzeoM4Y2Dq+qXzuJFA2sOpNgd71LxWsLMd6rowcXAggwrD/DM86pYXpNiRCDZfp0cDVSs= ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:06.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDhXBRxXVM1oEqmX7V0b34nPDO9KZAeqUJyJSErCC6+Kviu1preO52pPZ8NZUFCZONkNzPMquVFCGNcvu83+LLsrou8Mxq+WWiRPncu3nw1wpqP5JUNlFdtC8/v9QUBvK7Gx9f6BmLmI14slp2aBWqnrpPldNeSKFkjscZPZdkSd9FypMJNl5m/235Rz2au7WFoGjoxV66HTtDRtpjyKcpAqA0DvzOBmB+FGaKmhVjMe0oEhZMugdzjveceV9P6mGInBy7SfIZEqNa2eQl2PWPDCiazYNjotaXU7dxuOqai1GEsB5tayb9VNLHv0obsuYyE9oWUn9Alu3W/s1VI5waysbA2GUxghYDOSUbjYKoEmVFF6GOXqPZhu/M7aObulf6bYzlHcL4iOfaNJDLNToqiY5zGrHOupa1WcP5jKnRDPdgnNBlOAfltljOn9AzeoM4Y2Dq+qXzuJFA2sOpNgd71LxWsLMd6rowcXAggwrD/DM86pYXpNiRCDZfp0cDVSs= ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:02:06.788 INFO:teuthology.orchestra.run.smithi153.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDhXBRxXVM1oEqmX7V0b34nPDO9KZAeqUJyJSErCC6+Kviu1preO52pPZ8NZUFCZONkNzPMquVFCGNcvu83+LLsrou8Mxq+WWiRPncu3nw1wpqP5JUNlFdtC8/v9QUBvK7Gx9f6BmLmI14slp2aBWqnrpPldNeSKFkjscZPZdkSd9FypMJNl5m/235Rz2au7WFoGjoxV66HTtDRtpjyKcpAqA0DvzOBmB+FGaKmhVjMe0oEhZMugdzjveceV9P6mGInBy7SfIZEqNa2eQl2PWPDCiazYNjotaXU7dxuOqai1GEsB5tayb9VNLHv0obsuYyE9oWUn9Alu3W/s1VI5waysbA2GUxghYDOSUbjYKoEmVFF6GOXqPZhu/M7aObulf6bYzlHcL4iOfaNJDLNToqiY5zGrHOupa1WcP5jKnRDPdgnNBlOAfltljOn9AzeoM4Y2Dq+qXzuJFA2sOpNgd71LxWsLMd6rowcXAggwrD/DM86pYXpNiRCDZfp0cDVSs= ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:06.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T04:02:07.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:06 smithi082 bash[27462]: audit 2023-10-09T04:02:05.856403+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.82:0/3344818728' entity='client.admin' 2023-10-09T04:02:12.653 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T04:02:12.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T04:02:13.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:12 smithi082 bash[27462]: audit 2023-10-09T04:02:11.945945+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.82:0/3001562035' entity='client.admin' 2023-10-09T04:02:14.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:14 smithi082 bash[27462]: audit 2023-10-09T04:02:13.591813+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:14.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:14 smithi082 bash[27462]: audit 2023-10-09T04:02:13.862202+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi082", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:02:14.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:14 smithi082 bash[27462]: audit 2023-10-09T04:02:13.868210+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:14.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:14 smithi082 bash[27462]: audit 2023-10-09T04:02:13.870269+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:14 smithi082 bash[27462]: audit 2023-10-09T04:02:13.876477+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:16.162 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:15 smithi082 bash[27462]: cluster 2023-10-09T04:02:14.505561+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:16.615 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi153 2023-10-09T04:02:16.615 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:02:16.615 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.conf 2023-10-09T04:02:16.623 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:02:16.623 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:16.670 INFO:tasks.cephadm:Adding host smithi153 to orchestrator... 2023-10-09T04:02:16.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi153 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:15.903496+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:15.945533+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:15.948827+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:15.950364+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: cephadm 2023-10-09T04:02:15.952628+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: cephadm 2023-10-09T04:02:15.969975+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:16.001592+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: audit 2023-10-09T04:02:16.005550+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:16 smithi082 bash[27462]: cluster 2023-10-09T04:02:16.506068+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:19.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:19 smithi082 bash[27462]: cluster 2023-10-09T04:02:18.506643+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:20.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:02:19 smithi082 bash[27740]: Warning: Permanently added 'smithi153,172.21.15.153' (ECDSA) to the list of known hosts. 2023-10-09T04:02:20.561 INFO:teuthology.orchestra.run.smithi082.stdout:Added host 'smithi153' with addr '172.21.15.153' 2023-10-09T04:02:20.730 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:02:20 smithi082 bash[27740]: Warning: Permanently added '172.21.15.153' (ECDSA) to the list of known hosts. 2023-10-09T04:02:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:20 smithi082 bash[27462]: audit 2023-10-09T04:02:19.722973+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi153", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:21.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T04:02:21.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:21 smithi082 bash[27462]: cluster 2023-10-09T04:02:20.507208+0000 mgr.y (mgr.14146) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:21.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:21 smithi082 bash[27462]: audit 2023-10-09T04:02:20.559314+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:21.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:21 smithi082 bash[27462]: cephadm 2023-10-09T04:02:20.560015+0000 mgr.y (mgr.14146) 13 : cephadm [INF] Added host smithi153 2023-10-09T04:02:21.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:21 smithi082 bash[27462]: audit 2023-10-09T04:02:20.593668+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:21.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:21 smithi082 bash[27462]: audit 2023-10-09T04:02:21.142927+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:23.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:23 smithi082 bash[27462]: audit 2023-10-09T04:02:22.398370+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:23.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:23 smithi082 bash[27462]: cluster 2023-10-09T04:02:22.507790+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:24.326 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:02:24.326 INFO:teuthology.orchestra.run.smithi082.stdout:[{"addr": "172.21.15.82", "hostname": "smithi082", "labels": [], "status": ""}, {"addr": "172.21.15.153", "hostname": "smithi153", "labels": [], "status": ""}] 2023-10-09T04:02:25.029 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T04:02:25.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T04:02:25.860 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:25 smithi082 bash[27462]: audit 2023-10-09T04:02:24.324636+0000 mgr.y (mgr.14146) 15 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:02:25.860 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:25 smithi082 bash[27462]: cluster 2023-10-09T04:02:24.508211+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:27.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:27 smithi082 bash[27462]: cluster 2023-10-09T04:02:26.508703+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:28.592 INFO:teuthology.orchestra.run.smithi082.stderr:adjusted tunables profile to default 2023-10-09T04:02:28.851 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.581549+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.797514+0000 mon.a (mon.0) 106 : audit [INF] from='client.? 172.21.15.82:0/3117650326' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.880259+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.881632+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi153", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.883230+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.884405+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: cephadm 2023-10-09T04:02:27.886149+0000 mgr.y (mgr.14146) 18 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.938555+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:28.852 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:28 smithi082 bash[27462]: audit 2023-10-09T04:02:27.942263+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:29.266 INFO:tasks.cephadm:Adding mon.a on smithi082 2023-10-09T04:02:29.266 INFO:tasks.cephadm:Adding mon.c on smithi082 2023-10-09T04:02:29.266 INFO:tasks.cephadm:Adding mon.b on smithi153 2023-10-09T04:02:29.267 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi082:172.21.15.82=a;smithi082:[v2:172.21.15.82:3301,v1:172.21.15.82:6790]=c;smithi153:172.21.15.153=b' 2023-10-09T04:02:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:29 smithi082 bash[27462]: cephadm 2023-10-09T04:02:27.906472+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:29 smithi082 bash[27462]: cluster 2023-10-09T04:02:28.509127+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:29 smithi082 bash[27462]: audit 2023-10-09T04:02:28.586035+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.82:0/3117650326' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:02:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:29 smithi082 bash[27462]: cluster 2023-10-09T04:02:28.586169+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:31 smithi082 bash[27462]: cluster 2023-10-09T04:02:30.509473+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:32.174 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mon update... 2023-10-09T04:02:33.139 DEBUG:teuthology.orchestra.run.smithi082:mon.c> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T04:02:33.141 DEBUG:teuthology.orchestra.run.smithi153:mon.b> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T04:02:33.145 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:02:33.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:02:33.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:02:33.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.162217+0000 mgr.y (mgr.14146) 22 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi082:172.21.15.82=a;smithi082:[v2:172.21.15.82:3301,v1:172.21.15.82:6790]=c;smithi153:172.21.15.153=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:33.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: cephadm 2023-10-09T04:02:32.168402+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Saving service mon spec with placement smithi082:172.21.15.82=a;smithi082:[v2:172.21.15.82:3301,v1:172.21.15.82:6790]=c;smithi153:172.21.15.153=b;count:3 2023-10-09T04:02:33.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.172694+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:33.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.220140+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.223925+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.225703+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.232677+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.235468+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: audit 2023-10-09T04:02:32.236720+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: cephadm 2023-10-09T04:02:32.237824+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Deploying daemon mon.b on smithi153 2023-10-09T04:02:33.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:33 smithi082 bash[27462]: cluster 2023-10-09T04:02:32.509823+0000 mgr.y (mgr.14146) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:34.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:02:35.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:35 smithi082 bash[27462]: cluster 2023-10-09T04:02:34.510235+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:37.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:37 smithi082 bash[27462]: cluster 2023-10-09T04:02:36.510653+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:38.241 INFO:teuthology.orchestra.run.smithi153.stdout: 2023-10-09T04:02:38.241 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:00:53.174390Z","created":"2023-10-09T04:00:53.174390Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T04:02:38.244 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2023-10-09T04:02:38.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:38 smithi082 bash[27462]: audit 2023-10-09T04:02:38.240369+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.153:0/1434110085' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:39.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:39 smithi153 bash[27868]: debug 2023-10-09T04:02:39.802+0000 7fb65667e700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:02:39.934 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:02:39.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:02:42.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:42 smithi082 systemd[1]: Started Ceph mon.c for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:43.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.614+0000 7fe7fef34880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.614+0000 7fe7fef34880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.614+0000 7fe7fef34880 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 0 load: jerasure load: lrc load: isa 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: DB SUMMARY 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:02:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 511 ; 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.env: 0x5632543c1080 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.fs: Posix File System 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.info_log: 0x563254d67a80 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:02:43.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:02:43.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.db_log_dir: 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:02:43.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.write_buffer_manager: 0x563255a64360 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:02:43.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:02:43.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.row_cache: None 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.wal_filter: None 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:02:43.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:02:43.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Compression algorithms supported: 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:02:43.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kZSTD supported: 0 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.merge_operator: 2023-10-09T04:02:43.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563254d15d18) 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: cache_index_and_filter_blocks: 1 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: pin_top_level_index_and_filter: 1 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: index_type: 0 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: data_block_index_type: 0 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: index_shortening: 1 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:02:43.992 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: hash_index_allow_collision: 1 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: checksum: 1 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: no_block_cache: 0 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_cache: 0x563254d4cf10 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_cache_name: BinnedLRUCache 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_cache_options: 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: capacity : 536870912 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: num_shard_bits : 4 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: strict_capacity_limit : 0 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: high_pri_pool_ratio: 0.000 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_cache_compressed: (nil) 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: persistent_cache: (nil) 2023-10-09T04:02:43.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_size: 4096 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_size_deviation: 10 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_restart_interval: 16 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: index_block_restart_interval: 1 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: metadata_block_size: 4096 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: partition_filters: 0 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: use_delta_encoding: 1 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: whole_key_filtering: 1 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: verify_compression: 0 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: read_amp_bytes_per_bit: 0 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: format_version: 2 2023-10-09T04:02:43.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: enable_index_compression: 1 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: block_align: 0 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.num_levels: 7 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:02:43.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:02:43.996 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:02:43.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:02:43.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:02:43.999 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.618+0000 7fe7fef34880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824163625237, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T04:02:44.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824163626400, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696824163, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696824163627406, "job": 1, "event": "recovery_finished"} 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.622+0000 7fe7fef34880 4 rocksdb: DB pointer 0x563254e15800 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7e5d3b700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7e5d3b700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** DB Stats ** 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:44.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** Compaction Stats [default] ** 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** Compaction Stats [default] ** 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:44.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Cumulative compaction: 0.00 GB write, 0.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Interval compaction: 0.00 GB write, 0.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:44.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** Compaction Stats [default] ** 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** Compaction Stats [default] ** 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.3 0.00 0.00 1 0.001 0 0 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:44.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Cumulative compaction: 0.00 GB write, 0.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7fef34880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7fef34880 0 using public_addrv [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7fef34880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] at bind addrs [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.626+0000 7fe7fef34880 1 mon.c@-1(???) e0 preinit fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 0 mon.c@-1(synchronizing).mds e1 new map 2023-10-09T04:02:44.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: e1 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: legacy client fscid: -1 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: No filesystems configured 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T04:02:44.006 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.646+0000 7fe7e8540700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.650+0000 7fe7e8540700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.650+0000 7fe7e8540700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.650+0000 7fe7e8540700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.650+0000 7fe7e8540700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: audit 2023-10-09T04:02:38.240369+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.153:0/1434110085' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:44.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:43 smithi082 bash[33901]: debug 2023-10-09T04:02:43.650+0000 7fe7e8540700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:02:45.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:39.814196+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:45.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:39.814675+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:39.837865+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:40.511524+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:40.809550+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:41.809903+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:41.811169+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:42.511886+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:42.810074+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:43.658362+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:43.810344+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.512319+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:44.658580+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:44.810725+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.857748+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.868104+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.869028+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.869174+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.869439+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 50s) 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: cluster 2023-10-09T04:02:44.875173+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:44.881704+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:44 smithi082 bash[27462]: audit 2023-10-09T04:02:44.883943+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:39.814196+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:39.814675+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:39.837865+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:40.511524+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:40.809550+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:41.809903+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:41.811169+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:42.511886+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:42.810074+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:43.658362+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:43.810344+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.512319+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:44.658580+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:44.810725+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.857748+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.868104+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.869028+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.869174+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.869439+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 50s) 2023-10-09T04:02:45.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: cluster 2023-10-09T04:02:44.875173+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:45.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:44.881704+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:45.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:44 smithi153 bash[27868]: audit 2023-10-09T04:02:44.883943+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:46.230 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:02:45 smithi082 bash[27740]: debug 2023-10-09T04:02:45.806+0000 7fa6cea2a700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T04:02:50.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:45.664503+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:45.664981+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:45.670487+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:45.691036+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:45.691492+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:46.512706+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:46.658881+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:47.659274+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:47.660918+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:48.513173+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:48.659207+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:49.659147+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.659546+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.690444+0000 mon.a (mon.0) 155 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.712313+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.722366+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.741727+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.741913+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.742277+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 56s) 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.750025+0000 mon.a (mon.0) 161 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.753156+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.758412+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:39.814196+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:39.814675+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:50.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:39.837865+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:40.511524+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:40.809550+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:41.809903+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:41.811169+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:42.511886+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:42.810074+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:43.658362+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:43.810344+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.512319+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:44.658580+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:44.810725+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.857748+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.868104+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} 2023-10-09T04:02:50.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.869028+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.869174+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.869439+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 50s) 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:44.875173+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:44.881704+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.764205+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.765361+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:50.987 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[27462]: audit 2023-10-09T04:02:50.766771+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:44.883943+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:45.664503+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:45.664981+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:45.670487+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:45.691036+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:45.691492+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:46.512706+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:46.658881+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:47.659274+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:47.660918+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:48.513173+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:48.659207+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:49.659147+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.659546+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.690444+0000 mon.a (mon.0) 155 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.712313+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.722366+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.741727+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.741913+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.742277+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 56s) 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.750025+0000 mon.a (mon.0) 161 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.753156+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.758412+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.764205+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:50.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.765361+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:50.990 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:50 smithi082 bash[33901]: audit 2023-10-09T04:02:50.766771+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:45.664503+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:45.664981+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:45.670487+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:45.691036+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:45.691492+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:46.512706+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:46.658881+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:47.659274+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:47.660918+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:02:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:48.513173+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:48.659207+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:49.659147+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.659546+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.690444+0000 mon.a (mon.0) 155 : cluster [INF] mon.a calling monitor election 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.712313+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.722366+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.741727+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.741913+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.742277+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 56s) 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.750025+0000 mon.a (mon.0) 161 : cluster [INF] overall HEALTH_OK 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.753156+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.758412+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.764205+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.765361+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:50 smithi153 bash[27868]: audit 2023-10-09T04:02:50.766771+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:52.028 INFO:teuthology.orchestra.run.smithi153.stdout: 2023-10-09T04:02:52.029 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":3,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:02:45.659080Z","created":"2023-10-09T04:00:53.174390Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:3300","nonce":0},{"type":"v1","addr":"172.21.15.153:6789","nonce":0}]},"addr":"172.21.15.153:6789/0","public_addr":"172.21.15.153:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3301","nonce":0},{"type":"v1","addr":"172.21.15.82:6790","nonce":0}]},"addr":"172.21.15.82:6790/0","public_addr":"172.21.15.82:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T04:02:52.033 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 3 2023-10-09T04:02:52.082 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: cluster 2023-10-09T04:02:50.513532+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:52.082 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: cephadm 2023-10-09T04:02:50.768592+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:02:52.082 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: cephadm 2023-10-09T04:02:50.769020+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:02:52.082 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.796676+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.809254+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.813046+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: cephadm 2023-10-09T04:02:50.814686+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.815023+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.815982+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:50.816758+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: cephadm 2023-10-09T04:02:50.817546+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi082 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[27462]: audit 2023-10-09T04:02:51.659666+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: cluster 2023-10-09T04:02:50.513532+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:52.083 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: cephadm 2023-10-09T04:02:50.768592+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: cephadm 2023-10-09T04:02:50.769020+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.796676+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.809254+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.813046+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: cephadm 2023-10-09T04:02:50.814686+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.815023+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.815982+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:50.816758+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: cephadm 2023-10-09T04:02:50.817546+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi082 2023-10-09T04:02:52.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:51 smithi082 bash[33901]: audit 2023-10-09T04:02:51.659666+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:52.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: cluster 2023-10-09T04:02:50.513532+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: cephadm 2023-10-09T04:02:50.768592+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: cephadm 2023-10-09T04:02:50.769020+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.796676+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.809254+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.813046+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: cephadm 2023-10-09T04:02:50.814686+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.815023+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.815982+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:50.816758+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: cephadm 2023-10-09T04:02:50.817546+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi082 2023-10-09T04:02:52.087 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:51 smithi153 bash[27868]: audit 2023-10-09T04:02:51.659666+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:52.820 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T04:02:52.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T04:02:53.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:52 smithi153 bash[27868]: audit 2023-10-09T04:02:52.027876+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.153:0/2823207000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:53.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:52 smithi153 bash[27868]: audit 2023-10-09T04:02:52.757431+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:53.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:52 smithi153 bash[27868]: audit 2023-10-09T04:02:52.758814+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:53.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:52 smithi153 bash[27868]: audit 2023-10-09T04:02:52.759971+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:53.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:52 smithi153 bash[27868]: audit 2023-10-09T04:02:52.760855+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:53.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[33901]: audit 2023-10-09T04:02:52.027876+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.153:0/2823207000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[33901]: audit 2023-10-09T04:02:52.757431+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[33901]: audit 2023-10-09T04:02:52.758814+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[33901]: audit 2023-10-09T04:02:52.759971+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[33901]: audit 2023-10-09T04:02:52.760855+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[27462]: audit 2023-10-09T04:02:52.027876+0000 mon.a (mon.0) 174 : audit [DBG] from='client.? 172.21.15.153:0/2823207000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[27462]: audit 2023-10-09T04:02:52.757431+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[27462]: audit 2023-10-09T04:02:52.758814+0000 mon.a (mon.0) 176 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[27462]: audit 2023-10-09T04:02:52.759971+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:53.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:52 smithi082 bash[27462]: audit 2023-10-09T04:02:52.760855+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:54.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:53 smithi153 bash[27868]: cluster 2023-10-09T04:02:52.514001+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:53 smithi153 bash[27868]: cephadm 2023-10-09T04:02:52.758387+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:54.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:53 smithi153 bash[27868]: cephadm 2023-10-09T04:02:52.761728+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi082 2023-10-09T04:02:54.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[27462]: cluster 2023-10-09T04:02:52.514001+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[27462]: cephadm 2023-10-09T04:02:52.758387+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:54.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[27462]: cephadm 2023-10-09T04:02:52.761728+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi082 2023-10-09T04:02:54.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[33901]: cluster 2023-10-09T04:02:52.514001+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[33901]: cephadm 2023-10-09T04:02:52.758387+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:54.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:53 smithi082 bash[33901]: cephadm 2023-10-09T04:02:52.761728+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi082 2023-10-09T04:02:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:55 smithi153 bash[27868]: cluster 2023-10-09T04:02:54.514412+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:55 smithi153 bash[27868]: audit 2023-10-09T04:02:55.451136+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:55 smithi153 bash[27868]: audit 2023-10-09T04:02:55.452169+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:55 smithi153 bash[27868]: audit 2023-10-09T04:02:55.453205+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:55 smithi153 bash[27868]: audit 2023-10-09T04:02:55.454170+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:56.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[33901]: cluster 2023-10-09T04:02:54.514412+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:56.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[33901]: audit 2023-10-09T04:02:55.451136+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:56.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[33901]: audit 2023-10-09T04:02:55.452169+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:56.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[33901]: audit 2023-10-09T04:02:55.453205+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[33901]: audit 2023-10-09T04:02:55.454170+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[27462]: cluster 2023-10-09T04:02:54.514412+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[27462]: audit 2023-10-09T04:02:55.451136+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[27462]: audit 2023-10-09T04:02:55.452169+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[27462]: audit 2023-10-09T04:02:55.453205+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:56.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:55 smithi082 bash[27462]: audit 2023-10-09T04:02:55.454170+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:56.591 INFO:teuthology.orchestra.run.smithi082.stdout:# minimal ceph.conf for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:56.591 INFO:teuthology.orchestra.run.smithi082.stdout:[global] 2023-10-09T04:02:56.591 INFO:teuthology.orchestra.run.smithi082.stdout: fsid = 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:56.591 INFO:teuthology.orchestra.run.smithi082.stdout: mon_host = [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] 2023-10-09T04:02:56.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[33901]: cephadm 2023-10-09T04:02:55.451841+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:56.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[33901]: cephadm 2023-10-09T04:02:55.455180+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2023-10-09T04:02:56.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[33901]: audit 2023-10-09T04:02:56.590423+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.82:0/2794742389' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:56.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[27462]: cephadm 2023-10-09T04:02:55.451841+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:56.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[27462]: cephadm 2023-10-09T04:02:55.455180+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2023-10-09T04:02:56.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:56 smithi082 bash[27462]: audit 2023-10-09T04:02:56.590423+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.82:0/2794742389' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:57.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:56 smithi153 bash[27868]: cephadm 2023-10-09T04:02:55.451841+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:57.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:56 smithi153 bash[27868]: cephadm 2023-10-09T04:02:55.455180+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi153 2023-10-09T04:02:57.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:56 smithi153 bash[27868]: audit 2023-10-09T04:02:56.590423+0000 mon.a (mon.0) 183 : audit [DBG] from='client.? 172.21.15.82:0/2794742389' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:57.302 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T04:02:57.302 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:57.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:02:57.319 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:02:57.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:57.376 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:02:57.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:02:57.392 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:02:57.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:57.455 INFO:tasks.cephadm:Adding mgr.y on smithi082 2023-10-09T04:02:57.455 INFO:tasks.cephadm:Adding mgr.x on smithi153 2023-10-09T04:02:57.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi082=y;smithi153=x' 2023-10-09T04:02:58.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: cluster 2023-10-09T04:02:56.514859+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:58.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: audit 2023-10-09T04:02:57.240861+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:58.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: audit 2023-10-09T04:02:57.242180+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:58.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: audit 2023-10-09T04:02:57.244449+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: audit 2023-10-09T04:02:57.245559+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:57 smithi153 bash[27868]: audit 2023-10-09T04:02:57.250970+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:58.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: cluster 2023-10-09T04:02:56.514859+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:58.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: audit 2023-10-09T04:02:57.240861+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:58.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: audit 2023-10-09T04:02:57.242180+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:58.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: audit 2023-10-09T04:02:57.244449+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: audit 2023-10-09T04:02:57.245559+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[27462]: audit 2023-10-09T04:02:57.250970+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: cluster 2023-10-09T04:02:56.514859+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: audit 2023-10-09T04:02:57.240861+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: audit 2023-10-09T04:02:57.242180+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: audit 2023-10-09T04:02:57.244449+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: audit 2023-10-09T04:02:57.245559+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:57 smithi082 bash[33901]: audit 2023-10-09T04:02:57.250970+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:02:59.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:02:59 smithi082 bash[27462]: cluster 2023-10-09T04:02:58.515274+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:59.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:02:59 smithi082 bash[33901]: cluster 2023-10-09T04:02:58.515274+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:59.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:02:59 smithi153 bash[27868]: cluster 2023-10-09T04:02:58.515274+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:00.657 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mgr update... 2023-10-09T04:03:01.700 DEBUG:teuthology.orchestra.run.smithi153:mgr.x> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T04:03:01.703 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T04:03:01.703 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:03:01.703 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:03:01.710 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:03:01.710 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_1 2023-10-09T04:03:01.759 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:03:01.759 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:01.759 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 490 Links: 1 2023-10-09T04:03:01.760 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:01.760 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2023-10-09 03:56:23.769375226 +0000 2023-10-09T04:03:01.760 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2023-10-09 03:56:23.437382877 +0000 2023-10-09T04:03:01.760 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2023-10-09 03:56:23.437382877 +0000 2023-10-09T04:03:01.760 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: - 2023-10-09T04:03:01.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:03:01.812 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2023-10-09T04:03:01.812 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2023-10-09T04:03:01.813 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000442877 s, 1.2 MB/s 2023-10-09T04:03:01.813 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:03:01.865 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_2 2023-10-09T04:03:01.915 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:03:01.915 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 495 Links: 1 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2023-10-09 03:56:24.349361856 +0000 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2023-10-09 03:56:23.993370062 +0000 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2023-10-09 03:56:23.993370062 +0000 2023-10-09T04:03:01.916 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: - 2023-10-09T04:03:01.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:03:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: cluster 2023-10-09T04:03:00.515686+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.646546+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi082=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: cephadm 2023-10-09T04:03:00.650015+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi082=y;smithi153=x;count:2 2023-10-09T04:03:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.656021+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.676298+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.679127+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.680588+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.687636+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.689676+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.693906+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.695208+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: audit 2023-10-09T04:03:00.696568+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:01 smithi153 bash[27868]: cephadm 2023-10-09T04:03:00.697652+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: cluster 2023-10-09T04:03:00.515686+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.646546+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi082=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: cephadm 2023-10-09T04:03:00.650015+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi082=y;smithi153=x;count:2 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.656021+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.676298+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.679127+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.680588+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:01.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.687636+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.689676+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.693906+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.695208+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: audit 2023-10-09T04:03:00.696568+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[33901]: cephadm 2023-10-09T04:03:00.697652+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: cluster 2023-10-09T04:03:00.515686+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.646546+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi082=y;smithi153=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: cephadm 2023-10-09T04:03:00.650015+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi082=y;smithi153=x;count:2 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.656021+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.676298+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.679127+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.680588+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.687636+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.689676+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.693906+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.695208+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: audit 2023-10-09T04:03:00.696568+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:01.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:01 smithi082 bash[27462]: cephadm 2023-10-09T04:03:00.697652+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:03:01.970 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2023-10-09T04:03:01.970 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2023-10-09T04:03:01.970 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000406275 s, 1.3 MB/s 2023-10-09T04:03:01.970 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:03:02.021 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_3 2023-10-09T04:03:02.071 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:03:02.071 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.071 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 501 Links: 1 2023-10-09T04:03:02.071 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.071 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2023-10-09 03:56:24.589356324 +0000 2023-10-09T04:03:02.072 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2023-10-09 03:56:24.585356417 +0000 2023-10-09T04:03:02.072 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2023-10-09 03:56:24.585356417 +0000 2023-10-09T04:03:02.072 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: - 2023-10-09T04:03:02.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:03:02.124 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2023-10-09T04:03:02.124 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2023-10-09T04:03:02.124 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.00030374 s, 1.7 MB/s 2023-10-09T04:03:02.125 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:03:02.131 INFO:journalctl@ceph.mgr.x.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:03:02.177 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_4 2023-10-09T04:03:02.227 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:03:02.227 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2023-10-09 03:56:25.433336871 +0000 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2023-10-09 03:56:25.121344062 +0000 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2023-10-09 03:56:25.121344062 +0000 2023-10-09T04:03:02.228 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: - 2023-10-09T04:03:02.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:03:02.280 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2023-10-09T04:03:02.281 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2023-10-09T04:03:02.281 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000386686 s, 1.3 MB/s 2023-10-09T04:03:02.282 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:03:02.333 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:03:02.333 DEBUG:teuthology.orchestra.run.smithi153:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:03:02.341 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:03:02.341 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_1 2023-10-09T04:03:02.390 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:03:02.390 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.390 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 5h/5d Inode: 490 Links: 1 2023-10-09T04:03:02.391 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.391 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2023-10-09 03:56:24.872953152 +0000 2023-10-09T04:03:02.391 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2023-10-09 03:56:24.524961177 +0000 2023-10-09T04:03:02.391 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2023-10-09 03:56:24.524961177 +0000 2023-10-09T04:03:02.391 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2023-10-09T04:03:02.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:03:02.446 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2023-10-09T04:03:02.446 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2023-10-09T04:03:02.446 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000390416 s, 1.3 MB/s 2023-10-09T04:03:02.447 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:03:02.500 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_2 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2023-10-09 03:56:25.384941344 +0000 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2023-10-09 03:56:25.060948816 +0000 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2023-10-09 03:56:25.060948816 +0000 2023-10-09T04:03:02.550 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2023-10-09T04:03:02.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:03:02.603 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2023-10-09T04:03:02.603 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2023-10-09T04:03:02.603 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000372909 s, 1.4 MB/s 2023-10-09T04:03:02.605 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:03:02.655 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_3 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 5h/5d Inode: 501 Links: 1 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2023-10-09 03:56:25.936928615 +0000 2023-10-09T04:03:02.708 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2023-10-09 03:56:25.608936180 +0000 2023-10-09T04:03:02.709 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2023-10-09 03:56:25.608936180 +0000 2023-10-09T04:03:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2023-10-09T04:03:02.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:03:02.761 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2023-10-09T04:03:02.761 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2023-10-09T04:03:02.762 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000409362 s, 1.3 MB/s 2023-10-09T04:03:02.762 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:03:02.811 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_4 2023-10-09T04:03:02.860 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:03:02.860 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 5h/5d Inode: 507 Links: 1 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2023-10-09 03:56:26.476916162 +0000 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2023-10-09 03:56:26.160923449 +0000 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2023-10-09 03:56:26.160923449 +0000 2023-10-09T04:03:02.861 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2023-10-09T04:03:02.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:03:02.914 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2023-10-09T04:03:02.914 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2023-10-09T04:03:02.914 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000446945 s, 1.1 MB/s 2023-10-09T04:03:02.915 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:03:02.968 INFO:tasks.cephadm:Deploying osd.0 on smithi082 with /dev/vg_nvme/lv_4... 2023-10-09T04:03:02.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:03:03.377 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:03 smithi153 systemd[1]: Started Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:03:03.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:03 smithi153 bash[27868]: cluster 2023-10-09T04:03:02.516130+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:03.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:03 smithi153 bash[27868]: audit 2023-10-09T04:03:03.348392+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:03.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:03 smithi153 bash[27868]: audit 2023-10-09T04:03:03.350990+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:03.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[33901]: cluster 2023-10-09T04:03:02.516130+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:03.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[33901]: audit 2023-10-09T04:03:03.348392+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:03.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[33901]: audit 2023-10-09T04:03:03.350990+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[27462]: cluster 2023-10-09T04:03:02.516130+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[27462]: audit 2023-10-09T04:03:03.348392+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:03 smithi082 bash[27462]: audit 2023-10-09T04:03:03.350990+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:04.424 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:04 smithi153 bash[28717]: debug 2023-10-09T04:03:04.282+0000 7ffa6fccf700 1 -- 172.21.15.153:0/585088696 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56083d3144e0 con 0x56083d409000 2023-10-09T04:03:04.924 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:04 smithi153 bash[28717]: debug 2023-10-09T04:03:04.442+0000 7ffa802b5680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:03:04.924 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:04 smithi153 bash[28717]: debug 2023-10-09T04:03:04.546+0000 7ffa802b5680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:03:05.424 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:05 smithi153 bash[28717]: debug 2023-10-09T04:03:05.014+0000 7ffa802b5680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:03:05.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:05 smithi153 bash[27868]: cluster 2023-10-09T04:03:04.516580+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:05.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:05 smithi082 bash[27462]: cluster 2023-10-09T04:03:04.516580+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:05.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:05 smithi082 bash[33901]: cluster 2023-10-09T04:03:04.516580+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10+0 records in 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr:10+0 records out 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0156916 s, 668 MB/s 2023-10-09T04:03:06.256 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping successful for: 2023-10-09T04:03:06.282 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:05 smithi153 bash[28717]: debug 2023-10-09T04:03:05.926+0000 7ffa802b5680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:03:06.282 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:06 smithi153 bash[28717]: debug 2023-10-09T04:03:06.018+0000 7ffa802b5680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:03:06.282 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:06 smithi153 bash[28717]: debug 2023-10-09T04:03:06.278+0000 7ffa802b5680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:03:06.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi082:vg_nvme/lv_4 2023-10-09T04:03:07.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:06 smithi082 bash[27462]: audit 2023-10-09T04:03:05.759950+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:07.083 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:06 smithi082 bash[33901]: audit 2023-10-09T04:03:05.759950+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:07.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:06 smithi153 bash[27868]: audit 2023-10-09T04:03:05.759950+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:07.674 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:07 smithi153 bash[28717]: debug 2023-10-09T04:03:07.414+0000 7ffa802b5680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:03:07.674 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:07 smithi153 bash[28717]: debug 2023-10-09T04:03:07.574+0000 7ffa802b5680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:03:08.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:07 smithi153 bash[27868]: cluster 2023-10-09T04:03:06.516925+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:08.008 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:07 smithi153 bash[28717]: debug 2023-10-09T04:03:07.670+0000 7ffa802b5680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:03:08.008 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:07 smithi153 bash[28717]: debug 2023-10-09T04:03:07.782+0000 7ffa802b5680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:03:08.193 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:07 smithi082 bash[33901]: cluster 2023-10-09T04:03:06.516925+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:08.193 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:07 smithi082 bash[27462]: cluster 2023-10-09T04:03:06.516925+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:08.414 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:08 smithi153 bash[28717]: debug 2023-10-09T04:03:08.002+0000 7ffa802b5680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:03:08.674 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:08 smithi153 bash[28717]: debug 2023-10-09T04:03:08.410+0000 7ffa802b5680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:03:08.674 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:08 smithi153 bash[28717]: debug 2023-10-09T04:03:08.546+0000 7ffa802b5680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:03:09.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.420069+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.421314+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.422482+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:09.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.427967+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: cephadm 2023-10-09T04:03:08.430530+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.430951+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.432141+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: audit 2023-10-09T04:03:08.433133+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: cephadm 2023-10-09T04:03:08.433960+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[27868]: cluster 2023-10-09T04:03:08.517240+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[28717]: debug 2023-10-09T04:03:09.430+0000 7ffa802b5680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[28717]: debug 2023-10-09T04:03:09.534+0000 7ffa802b5680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:03:09.675 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[28717]: debug 2023-10-09T04:03:09.650+0000 7ffa802b5680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:03:09.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.420069+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.421314+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.422482+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.427967+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: cephadm 2023-10-09T04:03:08.430530+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.430951+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.432141+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: audit 2023-10-09T04:03:08.433133+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: cephadm 2023-10-09T04:03:08.433960+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[33901]: cluster 2023-10-09T04:03:08.517240+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.420069+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.421314+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.422482+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.427967+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: cephadm 2023-10-09T04:03:08.430530+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.430951+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.432141+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: audit 2023-10-09T04:03:08.433133+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: cephadm 2023-10-09T04:03:08.433960+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:03:09.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:09 smithi082 bash[27462]: cluster 2023-10-09T04:03:08.517240+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:10.160 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:09 smithi153 bash[28717]: debug 2023-10-09T04:03:09.862+0000 7ffa802b5680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:03:10.424 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[28717]: debug 2023-10-09T04:03:10.158+0000 7ffa802b5680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:03:10.424 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[28717]: debug 2023-10-09T04:03:10.322+0000 7ffa802b5680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:03:10.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[27462]: audit 2023-10-09T04:03:10.192559+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[27462]: audit 2023-10-09T04:03:10.195911+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[27462]: audit 2023-10-09T04:03:10.196643+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:10.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[33901]: audit 2023-10-09T04:03:10.192559+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:10.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[33901]: audit 2023-10-09T04:03:10.195911+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:10.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:10 smithi082 bash[33901]: audit 2023-10-09T04:03:10.196643+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:10.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[27868]: audit 2023-10-09T04:03:10.192559+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:10.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[27868]: audit 2023-10-09T04:03:10.195911+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:10.818 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[27868]: audit 2023-10-09T04:03:10.196643+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:10.818 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[28717]: debug 2023-10-09T04:03:10.498+0000 7ffa802b5680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:03:10.818 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[28717]: debug 2023-10-09T04:03:10.710+0000 7ffa802b5680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:03:11.174 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:03:10 smithi153 bash[28717]: debug 2023-10-09T04:03:10.814+0000 7ffa802b5680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:03:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.185436+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: cluster 2023-10-09T04:03:10.517583+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.668588+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.670231+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.672431+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.673553+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.679058+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: cluster 2023-10-09T04:03:10.839604+0000 mon.a (mon.0) 216 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.842515+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.843448+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.845601+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:03:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:11 smithi153 bash[27868]: audit 2023-10-09T04:03:10.846219+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.185436+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: cluster 2023-10-09T04:03:10.517583+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.668588+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.670231+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.672431+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.673553+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.679058+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: cluster 2023-10-09T04:03:10.839604+0000 mon.a (mon.0) 216 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.842515+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.843448+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.845601+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[27462]: audit 2023-10-09T04:03:10.846219+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.185436+0000 mgr.y (mgr.14146) 58 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: cluster 2023-10-09T04:03:10.517583+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.668588+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.670231+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.672431+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.673553+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.679058+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: cluster 2023-10-09T04:03:10.839604+0000 mon.a (mon.0) 216 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.842515+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.843448+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.845601+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:03:11.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:11 smithi082 bash[33901]: audit 2023-10-09T04:03:10.846219+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.153:0/2309140509' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:03:12.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:12 smithi082 bash[27462]: cluster 2023-10-09T04:03:11.685402+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T04:03:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:12 smithi082 bash[27462]: audit 2023-10-09T04:03:11.685582+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:03:12.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:12 smithi082 bash[33901]: cluster 2023-10-09T04:03:11.685402+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T04:03:12.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:12 smithi082 bash[33901]: audit 2023-10-09T04:03:11.685582+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:03:13.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:12 smithi153 bash[27868]: cluster 2023-10-09T04:03:11.685402+0000 mon.a (mon.0) 217 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T04:03:13.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:12 smithi153 bash[27868]: audit 2023-10-09T04:03:11.685582+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:03:13.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:13 smithi082 bash[27462]: cluster 2023-10-09T04:03:12.518012+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:13.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:13 smithi082 bash[33901]: cluster 2023-10-09T04:03:12.518012+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:14.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:13 smithi153 bash[27868]: cluster 2023-10-09T04:03:12.518012+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:15.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:15 smithi082 bash[27462]: cluster 2023-10-09T04:03:14.518413+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:15.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:15 smithi082 bash[33901]: cluster 2023-10-09T04:03:14.518413+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:16.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:15 smithi153 bash[27868]: cluster 2023-10-09T04:03:14.518413+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:17.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:17 smithi082 bash[27462]: cluster 2023-10-09T04:03:16.518872+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:17.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:17 smithi082 bash[33901]: cluster 2023-10-09T04:03:16.518872+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:18.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:17 smithi153 bash[27868]: cluster 2023-10-09T04:03:16.518872+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:18.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[27462]: audit 2023-10-09T04:03:18.063938+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]: dispatch 2023-10-09T04:03:18.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[27462]: audit 2023-10-09T04:03:18.071127+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]': finished 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[27462]: cluster 2023-10-09T04:03:18.071263+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[27462]: audit 2023-10-09T04:03:18.071458+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[33901]: audit 2023-10-09T04:03:18.063938+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]: dispatch 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[33901]: audit 2023-10-09T04:03:18.071127+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]': finished 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[33901]: cluster 2023-10-09T04:03:18.071263+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:03:18.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:18 smithi082 bash[33901]: audit 2023-10-09T04:03:18.071458+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:19.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:18 smithi153 bash[27868]: audit 2023-10-09T04:03:18.063938+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]: dispatch 2023-10-09T04:03:19.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:18 smithi153 bash[27868]: audit 2023-10-09T04:03:18.071127+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.82:0/2437453385' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a068a4f9-387d-4d69-afa5-d160994babe9"}]': finished 2023-10-09T04:03:19.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:18 smithi153 bash[27868]: cluster 2023-10-09T04:03:18.071263+0000 mon.a (mon.0) 221 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:03:19.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:18 smithi153 bash[27868]: audit 2023-10-09T04:03:18.071458+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:19.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:19 smithi082 bash[27462]: cluster 2023-10-09T04:03:18.519299+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:19.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:19 smithi082 bash[27462]: audit 2023-10-09T04:03:18.771905+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.82:0/2005566191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:19 smithi082 bash[33901]: cluster 2023-10-09T04:03:18.519299+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:19 smithi082 bash[33901]: audit 2023-10-09T04:03:18.771905+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.82:0/2005566191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:20.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:19 smithi153 bash[27868]: cluster 2023-10-09T04:03:18.519299+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:20.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:19 smithi153 bash[27868]: audit 2023-10-09T04:03:18.771905+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.82:0/2005566191' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:21.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:21 smithi082 bash[27462]: cluster 2023-10-09T04:03:20.519809+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:21 smithi082 bash[33901]: cluster 2023-10-09T04:03:20.519809+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:22.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:21 smithi153 bash[27868]: cluster 2023-10-09T04:03:20.519809+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:23.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:23 smithi082 bash[27462]: cluster 2023-10-09T04:03:22.520343+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:23.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:23 smithi082 bash[33901]: cluster 2023-10-09T04:03:22.520343+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:24.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:23 smithi153 bash[27868]: cluster 2023-10-09T04:03:22.520343+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:25.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:25 smithi082 bash[27462]: cluster 2023-10-09T04:03:24.520819+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:25.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:25 smithi082 bash[33901]: cluster 2023-10-09T04:03:24.520819+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:26.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:25 smithi153 bash[27868]: cluster 2023-10-09T04:03:24.520819+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:27.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:27 smithi082 bash[27462]: cluster 2023-10-09T04:03:26.521277+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:27.755 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:27 smithi082 bash[33901]: cluster 2023-10-09T04:03:26.521277+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:28.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:27 smithi153 bash[27868]: cluster 2023-10-09T04:03:26.521277+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:29.846 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:29 smithi082 bash[27462]: cluster 2023-10-09T04:03:28.521699+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:29.846 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:29 smithi082 bash[33901]: cluster 2023-10-09T04:03:28.521699+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:29.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:29 smithi153 bash[27868]: cluster 2023-10-09T04:03:28.521699+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:30.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:30 smithi153 bash[27868]: audit 2023-10-09T04:03:30.374080+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:03:30.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:30 smithi153 bash[27868]: audit 2023-10-09T04:03:30.376087+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:30.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:30 smithi082 bash[27462]: audit 2023-10-09T04:03:30.374080+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:03:30.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:30 smithi082 bash[27462]: audit 2023-10-09T04:03:30.376087+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:30.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:30 smithi082 bash[33901]: audit 2023-10-09T04:03:30.374080+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:03:30.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:30 smithi082 bash[33901]: audit 2023-10-09T04:03:30.376087+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:31.686 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:31 smithi082 bash[33901]: cephadm 2023-10-09T04:03:30.377651+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi082 2023-10-09T04:03:31.686 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:31 smithi082 bash[33901]: cluster 2023-10-09T04:03:30.522086+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:31.687 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:31 smithi082 bash[27462]: cephadm 2023-10-09T04:03:30.377651+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi082 2023-10-09T04:03:31.687 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:31 smithi082 bash[27462]: cluster 2023-10-09T04:03:30.522086+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:31.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:31 smithi153 bash[27868]: cephadm 2023-10-09T04:03:30.377651+0000 mgr.y (mgr.14146) 69 : cephadm [INF] Deploying daemon osd.0 on smithi082 2023-10-09T04:03:31.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:31 smithi153 bash[27868]: cluster 2023-10-09T04:03:30.522086+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:33.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:33 smithi082 bash[27462]: cluster 2023-10-09T04:03:32.522522+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:33.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:33 smithi082 bash[33901]: cluster 2023-10-09T04:03:32.522522+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:33.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:33 smithi153 bash[27868]: cluster 2023-10-09T04:03:32.522522+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:35.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[33901]: audit 2023-10-09T04:03:34.421546+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:35.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[33901]: audit 2023-10-09T04:03:34.458849+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:35.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[33901]: cluster 2023-10-09T04:03:34.522938+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[27462]: audit 2023-10-09T04:03:34.421546+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[27462]: audit 2023-10-09T04:03:34.458849+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:35 smithi082 bash[27462]: cluster 2023-10-09T04:03:34.522938+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:35.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:35 smithi153 bash[27868]: audit 2023-10-09T04:03:34.421546+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:35.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:35 smithi153 bash[27868]: audit 2023-10-09T04:03:34.458849+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:35.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:35 smithi153 bash[27868]: cluster 2023-10-09T04:03:34.522938+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:37.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:37 smithi082 bash[27462]: cluster 2023-10-09T04:03:36.523402+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:37.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:37 smithi082 bash[33901]: cluster 2023-10-09T04:03:36.523402+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:37.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:37 smithi153 bash[27868]: cluster 2023-10-09T04:03:36.523402+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:39.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:39 smithi153 bash[27868]: cluster 2023-10-09T04:03:38.523928+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:39.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:39 smithi082 bash[27462]: cluster 2023-10-09T04:03:38.523928+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:39.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:39 smithi082 bash[33901]: cluster 2023-10-09T04:03:38.523928+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:40.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[27462]: audit 2023-10-09T04:03:39.541386+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:40.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[27462]: audit 2023-10-09T04:03:39.817307+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[27462]: audit 2023-10-09T04:03:39.818177+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[27462]: audit 2023-10-09T04:03:39.823779+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[33901]: audit 2023-10-09T04:03:39.541386+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[33901]: audit 2023-10-09T04:03:39.817307+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[33901]: audit 2023-10-09T04:03:39.818177+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:40 smithi082 bash[33901]: audit 2023-10-09T04:03:39.823779+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:40.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:40 smithi153 bash[27868]: audit 2023-10-09T04:03:39.541386+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:40.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:40 smithi153 bash[27868]: audit 2023-10-09T04:03:39.817307+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:40.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:40 smithi153 bash[27868]: audit 2023-10-09T04:03:39.818177+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:40.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:40 smithi153 bash[27868]: audit 2023-10-09T04:03:39.823779+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:41.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:41 smithi082 bash[27462]: cluster 2023-10-09T04:03:40.524346+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:41.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:41 smithi082 bash[27462]: audit 2023-10-09T04:03:41.121517+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:41 smithi082 bash[33901]: cluster 2023-10-09T04:03:40.524346+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:41 smithi082 bash[33901]: audit 2023-10-09T04:03:41.121517+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:42.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:41 smithi153 bash[27868]: cluster 2023-10-09T04:03:40.524346+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:42.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:41 smithi153 bash[27868]: audit 2023-10-09T04:03:41.121517+0000 mon.a (mon.0) 231 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:43.052 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 0 on host 'smithi082' 2023-10-09T04:03:43.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: audit 2023-10-09T04:03:41.830752+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:43.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: cluster 2023-10-09T04:03:41.830926+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:43.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: audit 2023-10-09T04:03:41.831576+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: audit 2023-10-09T04:03:41.832119+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: audit 2023-10-09T04:03:42.831940+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: cluster 2023-10-09T04:03:42.832124+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[33901]: audit 2023-10-09T04:03:42.832423+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: audit 2023-10-09T04:03:41.830752+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: cluster 2023-10-09T04:03:41.830926+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:43.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: audit 2023-10-09T04:03:41.831576+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: audit 2023-10-09T04:03:41.832119+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:03:43.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: audit 2023-10-09T04:03:42.831940+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:03:43.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: cluster 2023-10-09T04:03:42.832124+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:43.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:42 smithi082 bash[27462]: audit 2023-10-09T04:03:42.832423+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: audit 2023-10-09T04:03:41.830752+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:43.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: cluster 2023-10-09T04:03:41.830926+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:43.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: audit 2023-10-09T04:03:41.831576+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: audit 2023-10-09T04:03:41.832119+0000 mon.a (mon.0) 235 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: audit 2023-10-09T04:03:42.831940+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: cluster 2023-10-09T04:03:42.832124+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:42 smithi153 bash[27868]: audit 2023-10-09T04:03:42.832423+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:43.782 DEBUG:teuthology.orchestra.run.smithi082:osd.0> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T04:03:43.785 INFO:tasks.cephadm:Deploying osd.1 on smithi082 with /dev/vg_nvme/lv_3... 2023-10-09T04:03:43.786 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:03:43.949 INFO:journalctl@ceph.osd.0.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:03:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:43 smithi153 bash[27868]: cluster 2023-10-09T04:03:42.524818+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:43 smithi153 bash[27868]: audit 2023-10-09T04:03:42.838780+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:43 smithi153 bash[27868]: audit 2023-10-09T04:03:43.048171+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:43 smithi153 bash[27868]: audit 2023-10-09T04:03:43.051782+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:43 smithi153 bash[27868]: audit 2023-10-09T04:03:43.838106+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:44.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[27462]: cluster 2023-10-09T04:03:42.524818+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:44.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[27462]: audit 2023-10-09T04:03:42.838780+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[27462]: audit 2023-10-09T04:03:43.048171+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[27462]: audit 2023-10-09T04:03:43.051782+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[27462]: audit 2023-10-09T04:03:43.838106+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[33901]: cluster 2023-10-09T04:03:42.524818+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[33901]: audit 2023-10-09T04:03:42.838780+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[33901]: audit 2023-10-09T04:03:43.048171+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[33901]: audit 2023-10-09T04:03:43.051782+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:43 smithi082 bash[33901]: audit 2023-10-09T04:03:43.838106+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:45.126 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[27462]: cluster 2023-10-09T04:03:42.166859+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:03:45.126 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[27462]: cluster 2023-10-09T04:03:42.166944+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[27462]: cluster 2023-10-09T04:03:43.856445+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838] boot 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[27462]: cluster 2023-10-09T04:03:43.856481+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[27462]: audit 2023-10-09T04:03:43.856791+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[33901]: cluster 2023-10-09T04:03:42.166859+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[33901]: cluster 2023-10-09T04:03:42.166944+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[33901]: cluster 2023-10-09T04:03:43.856445+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838] boot 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[33901]: cluster 2023-10-09T04:03:43.856481+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:45.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:44 smithi082 bash[33901]: audit 2023-10-09T04:03:43.856791+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:45.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:44 smithi153 bash[27868]: cluster 2023-10-09T04:03:42.166859+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:03:45.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:44 smithi153 bash[27868]: cluster 2023-10-09T04:03:42.166944+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:03:45.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:44 smithi153 bash[27868]: cluster 2023-10-09T04:03:43.856445+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.82:6802/2127587838,v1:172.21.15.82:6803/2127587838] boot 2023-10-09T04:03:45.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:44 smithi153 bash[27868]: cluster 2023-10-09T04:03:43.856481+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:45.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:44 smithi153 bash[27868]: audit 2023-10-09T04:03:43.856791+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:45.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:45 smithi082 bash[27462]: cluster 2023-10-09T04:03:44.525336+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:45.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:45 smithi082 bash[27462]: cluster 2023-10-09T04:03:44.863774+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:45.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:45 smithi082 bash[33901]: cluster 2023-10-09T04:03:44.525336+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:45.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:45 smithi082 bash[33901]: cluster 2023-10-09T04:03:44.863774+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:46.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:45 smithi153 bash[27868]: cluster 2023-10-09T04:03:44.525336+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:46.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:45 smithi153 bash[27868]: cluster 2023-10-09T04:03:44.863774+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:47.073 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:03:47.073 INFO:teuthology.orchestra.run.smithi082.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:03:47.073 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10+0 records in 2023-10-09T04:03:47.073 INFO:teuthology.orchestra.run.smithi082.stderr:10+0 records out 2023-10-09T04:03:47.074 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.024087 s, 435 MB/s 2023-10-09T04:03:47.074 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping successful for: 2023-10-09T04:03:47.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi082:vg_nvme/lv_3 2023-10-09T04:03:48.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:47 smithi153 bash[27868]: cluster 2023-10-09T04:03:46.525860+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:48.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:47 smithi082 bash[33901]: cluster 2023-10-09T04:03:46.525860+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:48.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:47 smithi082 bash[27462]: cluster 2023-10-09T04:03:46.525860+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:49.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:49 smithi082 bash[27462]: cluster 2023-10-09T04:03:48.526204+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:49.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:49 smithi082 bash[33901]: cluster 2023-10-09T04:03:48.526204+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:49.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:49 smithi153 bash[27868]: cluster 2023-10-09T04:03:48.526204+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:51.848 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:51 smithi082 bash[27462]: cluster 2023-10-09T04:03:50.526531+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:51.848 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:51 smithi082 bash[33901]: cluster 2023-10-09T04:03:50.526531+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:51.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:51 smithi153 bash[27868]: cluster 2023-10-09T04:03:50.526531+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:52.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:52 smithi153 bash[27868]: audit 2023-10-09T04:03:52.172338+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:52.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:52 smithi153 bash[27868]: audit 2023-10-09T04:03:52.175841+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:52.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:52 smithi153 bash[27868]: audit 2023-10-09T04:03:52.177085+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:52.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[27462]: audit 2023-10-09T04:03:52.172338+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:52.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[27462]: audit 2023-10-09T04:03:52.175841+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[27462]: audit 2023-10-09T04:03:52.177085+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:52.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[33901]: audit 2023-10-09T04:03:52.172338+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:52.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[33901]: audit 2023-10-09T04:03:52.175841+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:52.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:52 smithi082 bash[33901]: audit 2023-10-09T04:03:52.177085+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:52.169287+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:53.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: cluster 2023-10-09T04:03:52.526980+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:53.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.451004+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.452234+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.460246+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.461641+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.462866+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:53 smithi153 bash[27868]: audit 2023-10-09T04:03:53.471583+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:52.169287+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: cluster 2023-10-09T04:03:52.526980+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.451004+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.452234+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.460246+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.461641+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.462866+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[27462]: audit 2023-10-09T04:03:53.471583+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:52.169287+0000 mgr.y (mgr.14146) 81 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: cluster 2023-10-09T04:03:52.526980+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.451004+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.452234+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.460246+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.461641+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.462866+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:53 smithi082 bash[33901]: audit 2023-10-09T04:03:53.471583+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:03:54.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:54 smithi153 bash[27868]: cephadm 2023-10-09T04:03:53.444712+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:03:54.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:54 smithi153 bash[27868]: cephadm 2023-10-09T04:03:53.453229+0000 mgr.y (mgr.14146) 84 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 16241M 2023-10-09T04:03:54.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:54 smithi082 bash[27462]: cephadm 2023-10-09T04:03:53.444712+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:03:54.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:54 smithi082 bash[27462]: cephadm 2023-10-09T04:03:53.453229+0000 mgr.y (mgr.14146) 84 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 16241M 2023-10-09T04:03:54.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:54 smithi082 bash[33901]: cephadm 2023-10-09T04:03:53.444712+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:03:54.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:54 smithi082 bash[33901]: cephadm 2023-10-09T04:03:53.453229+0000 mgr.y (mgr.14146) 84 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 16241M 2023-10-09T04:03:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:55 smithi153 bash[27868]: cluster 2023-10-09T04:03:54.527600+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:55.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:55 smithi082 bash[27462]: cluster 2023-10-09T04:03:54.527600+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:55 smithi082 bash[33901]: cluster 2023-10-09T04:03:54.527600+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:57.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:57 smithi082 bash[27462]: cluster 2023-10-09T04:03:56.528087+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:57.876 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:57 smithi082 bash[33901]: cluster 2023-10-09T04:03:56.528087+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:57.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:57 smithi153 bash[27868]: cluster 2023-10-09T04:03:56.528087+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:59.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:03:59 smithi153 bash[27868]: cluster 2023-10-09T04:03:58.528534+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:59.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:03:59 smithi082 bash[27462]: cluster 2023-10-09T04:03:58.528534+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:59.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:03:59 smithi082 bash[33901]: cluster 2023-10-09T04:03:58.528534+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:00.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:00 smithi153 bash[27868]: audit 2023-10-09T04:04:00.108056+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.82:0/3207064184' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:00 smithi153 bash[27868]: audit 2023-10-09T04:04:00.108750+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:00 smithi153 bash[27868]: audit 2023-10-09T04:04:00.116234+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]': finished 2023-10-09T04:04:00.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:00 smithi153 bash[27868]: cluster 2023-10-09T04:04:00.116374+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:04:00.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:00 smithi153 bash[27868]: audit 2023-10-09T04:04:00.116565+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:00.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[27462]: audit 2023-10-09T04:04:00.108056+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.82:0/3207064184' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[27462]: audit 2023-10-09T04:04:00.108750+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[27462]: audit 2023-10-09T04:04:00.116234+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]': finished 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[27462]: cluster 2023-10-09T04:04:00.116374+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[27462]: audit 2023-10-09T04:04:00.116565+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[33901]: audit 2023-10-09T04:04:00.108056+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.82:0/3207064184' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[33901]: audit 2023-10-09T04:04:00.108750+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]: dispatch 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[33901]: audit 2023-10-09T04:04:00.116234+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fcfd2467-bc5b-4cf9-8fbb-71ee2b847070"}]': finished 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[33901]: cluster 2023-10-09T04:04:00.116374+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:04:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:00 smithi082 bash[33901]: audit 2023-10-09T04:04:00.116565+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:01 smithi153 bash[27868]: cluster 2023-10-09T04:04:00.528893+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:01 smithi153 bash[27868]: audit 2023-10-09T04:04:00.830163+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.82:0/3891264862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:01.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:01 smithi082 bash[27462]: cluster 2023-10-09T04:04:00.528893+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:01.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:01 smithi082 bash[27462]: audit 2023-10-09T04:04:00.830163+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.82:0/3891264862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:01 smithi082 bash[33901]: cluster 2023-10-09T04:04:00.528893+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:01 smithi082 bash[33901]: audit 2023-10-09T04:04:00.830163+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.82:0/3891264862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:03.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:03 smithi153 bash[27868]: cluster 2023-10-09T04:04:02.529419+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:03.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:03 smithi082 bash[27462]: cluster 2023-10-09T04:04:02.529419+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:03 smithi082 bash[33901]: cluster 2023-10-09T04:04:02.529419+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:05.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:05 smithi153 bash[27868]: cluster 2023-10-09T04:04:04.529897+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:05.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:05 smithi082 bash[27462]: cluster 2023-10-09T04:04:04.529897+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:05.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:05 smithi082 bash[33901]: cluster 2023-10-09T04:04:04.529897+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:07.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:07 smithi153 bash[27868]: cluster 2023-10-09T04:04:06.530426+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:07.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:07 smithi082 bash[33901]: cluster 2023-10-09T04:04:06.530426+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:07.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:07 smithi082 bash[27462]: cluster 2023-10-09T04:04:06.530426+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:09.835 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:09 smithi082 bash[33901]: cluster 2023-10-09T04:04:08.530934+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:09.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:09 smithi082 bash[27462]: cluster 2023-10-09T04:04:08.530934+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:09.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:09 smithi153 bash[27868]: cluster 2023-10-09T04:04:08.530934+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:11.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:11 smithi082 bash[33901]: cluster 2023-10-09T04:04:10.531418+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:11.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:11 smithi082 bash[27462]: cluster 2023-10-09T04:04:10.531418+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:11 smithi153 bash[27868]: cluster 2023-10-09T04:04:10.531418+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:12.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:12 smithi153 bash[27868]: audit 2023-10-09T04:04:12.503382+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:04:12.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:12 smithi153 bash[27868]: audit 2023-10-09T04:04:12.505027+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:12.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:12 smithi082 bash[27462]: audit 2023-10-09T04:04:12.503382+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:04:12.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:12 smithi082 bash[27462]: audit 2023-10-09T04:04:12.505027+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:12.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:12 smithi082 bash[33901]: audit 2023-10-09T04:04:12.503382+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:04:12.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:12 smithi082 bash[33901]: audit 2023-10-09T04:04:12.505027+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:13.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:13 smithi082 bash[27462]: cephadm 2023-10-09T04:04:12.506332+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi082 2023-10-09T04:04:13.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:13 smithi082 bash[27462]: cluster 2023-10-09T04:04:12.531934+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:13.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:13 smithi082 bash[33901]: cephadm 2023-10-09T04:04:12.506332+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi082 2023-10-09T04:04:13.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:13 smithi082 bash[33901]: cluster 2023-10-09T04:04:12.531934+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:13.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:13 smithi153 bash[27868]: cephadm 2023-10-09T04:04:12.506332+0000 mgr.y (mgr.14146) 94 : cephadm [INF] Deploying daemon osd.1 on smithi082 2023-10-09T04:04:13.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:13 smithi153 bash[27868]: cluster 2023-10-09T04:04:12.531934+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:15.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:15 smithi153 bash[27868]: cluster 2023-10-09T04:04:14.532498+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:15.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:15 smithi082 bash[27462]: cluster 2023-10-09T04:04:14.532498+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:15.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:15 smithi082 bash[33901]: cluster 2023-10-09T04:04:14.532498+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:17.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[27462]: cluster 2023-10-09T04:04:16.532963+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:17.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[27462]: audit 2023-10-09T04:04:16.788189+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[27462]: audit 2023-10-09T04:04:16.799803+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[33901]: cluster 2023-10-09T04:04:16.532963+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[33901]: audit 2023-10-09T04:04:16.788189+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:17 smithi082 bash[33901]: audit 2023-10-09T04:04:16.799803+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:18.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:17 smithi153 bash[27868]: cluster 2023-10-09T04:04:16.532963+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:18.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:17 smithi153 bash[27868]: audit 2023-10-09T04:04:16.788189+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:18.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:17 smithi153 bash[27868]: audit 2023-10-09T04:04:16.799803+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:19.798 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:19 smithi082 bash[33901]: cluster 2023-10-09T04:04:18.533436+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:19.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:19 smithi082 bash[27462]: cluster 2023-10-09T04:04:18.533436+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:19.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:19 smithi153 bash[27868]: cluster 2023-10-09T04:04:18.533436+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:21.815 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:21 smithi082 bash[27462]: cluster 2023-10-09T04:04:20.533944+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:21.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:21 smithi153 bash[27868]: cluster 2023-10-09T04:04:20.533944+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:22.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:21 smithi082 bash[33901]: cluster 2023-10-09T04:04:20.533944+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[27462]: audit 2023-10-09T04:04:21.930713+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[27462]: audit 2023-10-09T04:04:21.931878+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[27462]: audit 2023-10-09T04:04:21.932861+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[27462]: audit 2023-10-09T04:04:21.938401+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[27462]: cluster 2023-10-09T04:04:22.534442+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[33901]: audit 2023-10-09T04:04:21.930713+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[33901]: audit 2023-10-09T04:04:21.931878+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[33901]: audit 2023-10-09T04:04:21.932861+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[33901]: audit 2023-10-09T04:04:21.938401+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:22 smithi082 bash[33901]: cluster 2023-10-09T04:04:22.534442+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:23.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:22 smithi153 bash[27868]: audit 2023-10-09T04:04:21.930713+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:22 smithi153 bash[27868]: audit 2023-10-09T04:04:21.931878+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:23.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:22 smithi153 bash[27868]: audit 2023-10-09T04:04:21.932861+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:22 smithi153 bash[27868]: audit 2023-10-09T04:04:21.938401+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:22 smithi153 bash[27868]: cluster 2023-10-09T04:04:22.534442+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:24.201 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:23 smithi082 bash[27462]: audit 2023-10-09T04:04:23.321060+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:04:24.201 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:23 smithi082 bash[33901]: audit 2023-10-09T04:04:23.321060+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:04:24.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:23 smithi153 bash[27868]: audit 2023-10-09T04:04:23.321060+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:04:25.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[27462]: audit 2023-10-09T04:04:23.952572+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[27462]: cluster 2023-10-09T04:04:23.952698+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[27462]: audit 2023-10-09T04:04:23.952893+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[27462]: audit 2023-10-09T04:04:23.953317+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[27462]: cluster 2023-10-09T04:04:24.534915+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[33901]: audit 2023-10-09T04:04:23.952572+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[33901]: cluster 2023-10-09T04:04:23.952698+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[33901]: audit 2023-10-09T04:04:23.952893+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[33901]: audit 2023-10-09T04:04:23.953317+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:04:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:24 smithi082 bash[33901]: cluster 2023-10-09T04:04:24.534915+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:25.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:24 smithi153 bash[27868]: audit 2023-10-09T04:04:23.952572+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:04:25.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:24 smithi153 bash[27868]: cluster 2023-10-09T04:04:23.952698+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:04:25.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:24 smithi153 bash[27868]: audit 2023-10-09T04:04:23.952893+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:24 smithi153 bash[27868]: audit 2023-10-09T04:04:23.953317+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:04:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:24 smithi153 bash[27868]: cluster 2023-10-09T04:04:24.534915+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:25.501 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 1 on host 'smithi082' 2023-10-09T04:04:25.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: audit 2023-10-09T04:04:24.954729+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:04:25.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: cluster 2023-10-09T04:04:24.954897+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: audit 2023-10-09T04:04:24.955549+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: audit 2023-10-09T04:04:24.962381+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: audit 2023-10-09T04:04:25.499577+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[27462]: audit 2023-10-09T04:04:25.514215+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: audit 2023-10-09T04:04:24.954729+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: cluster 2023-10-09T04:04:24.954897+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: audit 2023-10-09T04:04:24.955549+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: audit 2023-10-09T04:04:24.962381+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:25.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: audit 2023-10-09T04:04:25.499577+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:25.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:25 smithi082 bash[33901]: audit 2023-10-09T04:04:25.514215+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:26.156 DEBUG:teuthology.orchestra.run.smithi082:osd.1> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T04:04:26.158 INFO:tasks.cephadm:Deploying osd.2 on smithi082 with /dev/vg_nvme/lv_2... 2023-10-09T04:04:26.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:04:26.375 INFO:journalctl@ceph.osd.1.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:04:26.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: audit 2023-10-09T04:04:24.954729+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:04:26.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: cluster 2023-10-09T04:04:24.954897+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:04:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: audit 2023-10-09T04:04:24.955549+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: audit 2023-10-09T04:04:24.962381+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: audit 2023-10-09T04:04:25.499577+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:25 smithi153 bash[27868]: audit 2023-10-09T04:04:25.514215+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:27.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: cluster 2023-10-09T04:04:24.319932+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: cluster 2023-10-09T04:04:24.320029+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: audit 2023-10-09T04:04:25.961329+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: cluster 2023-10-09T04:04:25.972931+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652] boot 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: cluster 2023-10-09T04:04:25.972973+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: audit 2023-10-09T04:04:25.973425+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[33901]: cluster 2023-10-09T04:04:26.535511+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: cluster 2023-10-09T04:04:24.319932+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: cluster 2023-10-09T04:04:24.320029+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:04:27.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: audit 2023-10-09T04:04:25.961329+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: cluster 2023-10-09T04:04:25.972931+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652] boot 2023-10-09T04:04:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: cluster 2023-10-09T04:04:25.972973+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:04:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: audit 2023-10-09T04:04:25.973425+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:26 smithi082 bash[27462]: cluster 2023-10-09T04:04:26.535511+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: cluster 2023-10-09T04:04:24.319932+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:04:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: cluster 2023-10-09T04:04:24.320029+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:04:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: audit 2023-10-09T04:04:25.961329+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: cluster 2023-10-09T04:04:25.972931+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.82:6810/2767022652,v1:172.21.15.82:6811/2767022652] boot 2023-10-09T04:04:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: cluster 2023-10-09T04:04:25.972973+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:04:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: audit 2023-10-09T04:04:25.973425+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:04:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:26 smithi153 bash[27868]: cluster 2023-10-09T04:04:26.535511+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:04:28.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:27 smithi153 bash[27868]: cluster 2023-10-09T04:04:26.983587+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:04:28.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:27 smithi082 bash[27462]: cluster 2023-10-09T04:04:26.983587+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:04:28.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:27 smithi082 bash[33901]: cluster 2023-10-09T04:04:26.983587+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10+0 records in 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr:10+0 records out 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0165409 s, 634 MB/s 2023-10-09T04:04:29.599 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping successful for: 2023-10-09T04:04:29.832 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:29 smithi082 bash[27462]: cluster 2023-10-09T04:04:28.535904+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:29.832 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:29 smithi082 bash[33901]: cluster 2023-10-09T04:04:28.535904+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:29.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:29 smithi153 bash[27868]: cluster 2023-10-09T04:04:28.535904+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:30.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi082:vg_nvme/lv_2 2023-10-09T04:04:31.850 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:31 smithi082 bash[33901]: cluster 2023-10-09T04:04:30.536294+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:31.850 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:31 smithi082 bash[27462]: cluster 2023-10-09T04:04:30.536294+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:31.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:31 smithi153 bash[27868]: cluster 2023-10-09T04:04:30.536294+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:33.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:33 smithi153 bash[27868]: cluster 2023-10-09T04:04:32.536796+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:33 smithi082 bash[27462]: cluster 2023-10-09T04:04:32.536796+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:33.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:33 smithi082 bash[33901]: cluster 2023-10-09T04:04:32.536796+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:35.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:35 smithi153 bash[27868]: cluster 2023-10-09T04:04:34.537272+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:35.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:35 smithi153 bash[27868]: audit 2023-10-09T04:04:34.646713+0000 mgr.y (mgr.14146) 107 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:35 smithi153 bash[27868]: audit 2023-10-09T04:04:34.649704+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:35 smithi153 bash[27868]: audit 2023-10-09T04:04:34.654104+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:35 smithi153 bash[27868]: audit 2023-10-09T04:04:34.655465+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[27462]: cluster 2023-10-09T04:04:34.537272+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[27462]: audit 2023-10-09T04:04:34.646713+0000 mgr.y (mgr.14146) 107 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[27462]: audit 2023-10-09T04:04:34.649704+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[27462]: audit 2023-10-09T04:04:34.654104+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[27462]: audit 2023-10-09T04:04:34.655465+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[33901]: cluster 2023-10-09T04:04:34.537272+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[33901]: audit 2023-10-09T04:04:34.646713+0000 mgr.y (mgr.14146) 107 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[33901]: audit 2023-10-09T04:04:34.649704+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[33901]: audit 2023-10-09T04:04:34.654104+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:35 smithi082 bash[33901]: audit 2023-10-09T04:04:34.655465+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:36.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: cephadm 2023-10-09T04:04:35.912614+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:04:36.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.920302+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.921712+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.923013+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.928819+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.930434+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.931429+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:36 smithi153 bash[27868]: audit 2023-10-09T04:04:35.937855+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: cephadm 2023-10-09T04:04:35.912614+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.920302+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.921712+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.923013+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.928819+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.930434+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.931429+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[27462]: audit 2023-10-09T04:04:35.937855+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: cephadm 2023-10-09T04:04:35.912614+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.920302+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.921712+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.923013+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.928819+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.930434+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.931429+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:36 smithi082 bash[33901]: audit 2023-10-09T04:04:35.937855+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:37.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:37 smithi153 bash[27868]: cephadm 2023-10-09T04:04:35.923830+0000 mgr.y (mgr.14146) 109 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 8120M 2023-10-09T04:04:37.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:37 smithi153 bash[27868]: cluster 2023-10-09T04:04:36.537794+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:37.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:37 smithi082 bash[27462]: cephadm 2023-10-09T04:04:35.923830+0000 mgr.y (mgr.14146) 109 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 8120M 2023-10-09T04:04:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:37 smithi082 bash[27462]: cluster 2023-10-09T04:04:36.537794+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:37 smithi082 bash[33901]: cephadm 2023-10-09T04:04:35.923830+0000 mgr.y (mgr.14146) 109 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 8120M 2023-10-09T04:04:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:37 smithi082 bash[33901]: cluster 2023-10-09T04:04:36.537794+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:39 smithi082 bash[27462]: cluster 2023-10-09T04:04:38.538321+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:39 smithi082 bash[33901]: cluster 2023-10-09T04:04:38.538321+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:39.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:39 smithi153 bash[27868]: cluster 2023-10-09T04:04:38.538321+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:41.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:41 smithi153 bash[27868]: cluster 2023-10-09T04:04:40.538793+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:41.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:41 smithi082 bash[27462]: cluster 2023-10-09T04:04:40.538793+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:41 smithi082 bash[33901]: cluster 2023-10-09T04:04:40.538793+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:43.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: cluster 2023-10-09T04:04:42.539266+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:43.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: audit 2023-10-09T04:04:42.681897+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]: dispatch 2023-10-09T04:04:43.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: audit 2023-10-09T04:04:42.688567+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]': finished 2023-10-09T04:04:43.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: cluster 2023-10-09T04:04:42.688646+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:04:43.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: audit 2023-10-09T04:04:42.688830+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:04:43.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:43 smithi153 bash[27868]: audit 2023-10-09T04:04:43.456620+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.82:0/3133904609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: cluster 2023-10-09T04:04:42.539266+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: audit 2023-10-09T04:04:42.681897+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]: dispatch 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: audit 2023-10-09T04:04:42.688567+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]': finished 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: cluster 2023-10-09T04:04:42.688646+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: audit 2023-10-09T04:04:42.688830+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:04:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[27462]: audit 2023-10-09T04:04:43.456620+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.82:0/3133904609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: cluster 2023-10-09T04:04:42.539266+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: audit 2023-10-09T04:04:42.681897+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]: dispatch 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: audit 2023-10-09T04:04:42.688567+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.82:0/3544468443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0b236da8-a93d-4c97-a234-b9033c7c7067"}]': finished 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: cluster 2023-10-09T04:04:42.688646+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: audit 2023-10-09T04:04:42.688830+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:04:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:43 smithi082 bash[33901]: audit 2023-10-09T04:04:43.456620+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.82:0/3133904609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:45.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:45 smithi153 bash[27868]: cluster 2023-10-09T04:04:44.539679+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:45.945 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:45 smithi082 bash[33901]: cluster 2023-10-09T04:04:44.539679+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:45.945 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:45 smithi082 bash[27462]: cluster 2023-10-09T04:04:44.539679+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:47.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:47 smithi153 bash[27868]: cluster 2023-10-09T04:04:46.540245+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:47.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:47 smithi082 bash[27462]: cluster 2023-10-09T04:04:46.540245+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:47.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:47 smithi082 bash[33901]: cluster 2023-10-09T04:04:46.540245+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:49.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:49 smithi153 bash[27868]: cluster 2023-10-09T04:04:48.540749+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:49.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:49 smithi082 bash[27462]: cluster 2023-10-09T04:04:48.540749+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:49.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:49 smithi082 bash[33901]: cluster 2023-10-09T04:04:48.540749+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:51.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:51 smithi082 bash[27462]: cluster 2023-10-09T04:04:50.541247+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:51.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:51 smithi082 bash[33901]: cluster 2023-10-09T04:04:50.541247+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:51.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:51 smithi153 bash[27868]: cluster 2023-10-09T04:04:50.541247+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:53.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:53 smithi153 bash[27868]: cluster 2023-10-09T04:04:52.541705+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:53.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:53 smithi082 bash[27462]: cluster 2023-10-09T04:04:52.541705+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:53.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:53 smithi082 bash[33901]: cluster 2023-10-09T04:04:52.541705+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[27462]: cluster 2023-10-09T04:04:54.542114+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[27462]: audit 2023-10-09T04:04:54.958707+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[27462]: audit 2023-10-09T04:04:54.960018+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[33901]: cluster 2023-10-09T04:04:54.542114+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[33901]: audit 2023-10-09T04:04:54.958707+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:04:55.894 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:55 smithi082 bash[33901]: audit 2023-10-09T04:04:54.960018+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:55 smithi153 bash[27868]: cluster 2023-10-09T04:04:54.542114+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:55 smithi153 bash[27868]: audit 2023-10-09T04:04:54.958707+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:04:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:55 smithi153 bash[27868]: audit 2023-10-09T04:04:54.960018+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:56.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:56 smithi153 bash[27868]: cephadm 2023-10-09T04:04:54.961200+0000 mgr.y (mgr.14146) 120 : cephadm [INF] Deploying daemon osd.2 on smithi082 2023-10-09T04:04:56.995 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:56 smithi082 bash[33901]: cephadm 2023-10-09T04:04:54.961200+0000 mgr.y (mgr.14146) 120 : cephadm [INF] Deploying daemon osd.2 on smithi082 2023-10-09T04:04:56.995 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:56 smithi082 bash[27462]: cephadm 2023-10-09T04:04:54.961200+0000 mgr.y (mgr.14146) 120 : cephadm [INF] Deploying daemon osd.2 on smithi082 2023-10-09T04:04:57.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:57 smithi082 bash[33901]: cluster 2023-10-09T04:04:56.542652+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:57.864 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:57 smithi082 bash[27462]: cluster 2023-10-09T04:04:56.542652+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:57.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:57 smithi153 bash[27868]: cluster 2023-10-09T04:04:56.542652+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:59.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[27462]: cluster 2023-10-09T04:04:58.543053+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:59.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[27462]: audit 2023-10-09T04:04:58.955371+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:59.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[27462]: audit 2023-10-09T04:04:58.968577+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:59.780 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[33901]: cluster 2023-10-09T04:04:58.543053+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:59.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[33901]: audit 2023-10-09T04:04:58.955371+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:59.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:04:59 smithi082 bash[33901]: audit 2023-10-09T04:04:58.968577+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:59.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:59 smithi153 bash[27868]: cluster 2023-10-09T04:04:58.543053+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:04:59.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:59 smithi153 bash[27868]: audit 2023-10-09T04:04:58.955371+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:04:59.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:04:59 smithi153 bash[27868]: audit 2023-10-09T04:04:58.968577+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:01.883 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:01 smithi082 bash[27462]: cluster 2023-10-09T04:05:00.543512+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:01.884 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:01 smithi082 bash[33901]: cluster 2023-10-09T04:05:00.543512+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:01 smithi153 bash[27868]: cluster 2023-10-09T04:05:00.543512+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:03.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:03 smithi153 bash[27868]: cluster 2023-10-09T04:05:02.544023+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:03.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:03 smithi082 bash[27462]: cluster 2023-10-09T04:05:02.544023+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:03 smithi082 bash[33901]: cluster 2023-10-09T04:05:02.544023+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:05.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:05 smithi153 bash[27868]: audit 2023-10-09T04:05:04.169494+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:05 smithi153 bash[27868]: audit 2023-10-09T04:05:04.458400+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:05 smithi153 bash[27868]: audit 2023-10-09T04:05:04.459685+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:05 smithi153 bash[27868]: audit 2023-10-09T04:05:04.465789+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:05 smithi153 bash[27868]: cluster 2023-10-09T04:05:04.544356+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:05.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[27462]: audit 2023-10-09T04:05:04.169494+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[27462]: audit 2023-10-09T04:05:04.458400+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[27462]: audit 2023-10-09T04:05:04.459685+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[27462]: audit 2023-10-09T04:05:04.465789+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[27462]: cluster 2023-10-09T04:05:04.544356+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[33901]: audit 2023-10-09T04:05:04.169494+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[33901]: audit 2023-10-09T04:05:04.458400+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:05.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[33901]: audit 2023-10-09T04:05:04.459685+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:05.456 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[33901]: audit 2023-10-09T04:05:04.465789+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:05.456 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:05 smithi082 bash[33901]: cluster 2023-10-09T04:05:04.544356+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:06 smithi082 bash[27462]: audit 2023-10-09T04:05:05.502131+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:05:06.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:06 smithi082 bash[33901]: audit 2023-10-09T04:05:05.502131+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:05:06.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:06 smithi153 bash[27868]: audit 2023-10-09T04:05:05.502131+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:05:07.617 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 2 on host 'smithi082' 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[27462]: audit 2023-10-09T04:05:06.476167+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[27462]: cluster 2023-10-09T04:05:06.476251+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[27462]: audit 2023-10-09T04:05:06.476404+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[27462]: audit 2023-10-09T04:05:06.476929+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[27462]: cluster 2023-10-09T04:05:06.544653+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[33901]: audit 2023-10-09T04:05:06.476167+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[33901]: cluster 2023-10-09T04:05:06.476251+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:05:07.823 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[33901]: audit 2023-10-09T04:05:06.476404+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:07.824 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[33901]: audit 2023-10-09T04:05:06.476929+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:07.824 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:07 smithi082 bash[33901]: cluster 2023-10-09T04:05:06.544653+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:07.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:07 smithi153 bash[27868]: audit 2023-10-09T04:05:06.476167+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:05:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:07 smithi153 bash[27868]: cluster 2023-10-09T04:05:06.476251+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:05:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:07 smithi153 bash[27868]: audit 2023-10-09T04:05:06.476404+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:07 smithi153 bash[27868]: audit 2023-10-09T04:05:06.476929+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:07 smithi153 bash[27868]: cluster 2023-10-09T04:05:06.544653+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:08.299 DEBUG:teuthology.orchestra.run.smithi082:osd.2> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T04:05:08.302 INFO:tasks.cephadm:Deploying osd.3 on smithi082 with /dev/vg_nvme/lv_1... 2023-10-09T04:05:08.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:05:08.531 INFO:journalctl@ceph.osd.2.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:05:09.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: audit 2023-10-09T04:05:07.480228+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:09.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: cluster 2023-10-09T04:05:07.480400+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:05:09.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: audit 2023-10-09T04:05:07.481252+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: audit 2023-10-09T04:05:07.486917+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: audit 2023-10-09T04:05:07.615378+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[33901]: audit 2023-10-09T04:05:07.641556+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: audit 2023-10-09T04:05:07.480228+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: cluster 2023-10-09T04:05:07.480400+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: audit 2023-10-09T04:05:07.481252+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: audit 2023-10-09T04:05:07.486917+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: audit 2023-10-09T04:05:07.615378+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:09.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:08 smithi082 bash[27462]: audit 2023-10-09T04:05:07.641556+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:09.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: audit 2023-10-09T04:05:07.480228+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:09.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: cluster 2023-10-09T04:05:07.480400+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:05:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: audit 2023-10-09T04:05:07.481252+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: audit 2023-10-09T04:05:07.486917+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: audit 2023-10-09T04:05:07.615378+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:08 smithi153 bash[27868]: audit 2023-10-09T04:05:07.641556+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:06.501821+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:06.501898+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:08.487668+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550] boot 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:08.487704+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: audit 2023-10-09T04:05:08.488261+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: audit 2023-10-09T04:05:08.488737+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:08.545019+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:10.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: audit 2023-10-09T04:05:09.496038+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: cluster 2023-10-09T04:05:09.496120+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[27462]: audit 2023-10-09T04:05:09.497275+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:06.501821+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:06.501898+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:08.487668+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550] boot 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:08.487704+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: audit 2023-10-09T04:05:08.488261+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: audit 2023-10-09T04:05:08.488737+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:08.545019+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:10.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: audit 2023-10-09T04:05:09.496038+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:05:10.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: cluster 2023-10-09T04:05:09.496120+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:05:10.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:09 smithi082 bash[33901]: audit 2023-10-09T04:05:09.497275+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:05:10.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:06.501821+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:06.501898+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:08.487668+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.82:6818/3755541550,v1:172.21.15.82:6819/3755541550] boot 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:08.487704+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: audit 2023-10-09T04:05:08.488261+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: audit 2023-10-09T04:05:08.488737+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:08.545019+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: audit 2023-10-09T04:05:09.496038+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: cluster 2023-10-09T04:05:09.496120+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:05:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:09 smithi153 bash[27868]: audit 2023-10-09T04:05:09.497275+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10+0 records in 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr:10+0 records out 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.018056 s, 581 MB/s 2023-10-09T04:05:11.643 INFO:teuthology.orchestra.run.smithi082.stderr:--> Zapping successful for: 2023-10-09T04:05:11.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[33901]: audit 2023-10-09T04:05:10.499666+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:05:11.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[33901]: cluster 2023-10-09T04:05:10.499819+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:05:11.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[33901]: cluster 2023-10-09T04:05:10.545500+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v108: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:11.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[27462]: audit 2023-10-09T04:05:10.499666+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:05:11.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[27462]: cluster 2023-10-09T04:05:10.499819+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:05:11.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:11 smithi082 bash[27462]: cluster 2023-10-09T04:05:10.545500+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v108: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:11 smithi153 bash[27868]: audit 2023-10-09T04:05:10.499666+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:05:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:11 smithi153 bash[27868]: cluster 2023-10-09T04:05:10.499819+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:05:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:11 smithi153 bash[27868]: cluster 2023-10-09T04:05:10.545500+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v108: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:12.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi082:vg_nvme/lv_1 2023-10-09T04:05:12.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:12 smithi082 bash[33901]: cluster 2023-10-09T04:05:11.507677+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:05:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:12 smithi082 bash[27462]: cluster 2023-10-09T04:05:11.507677+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:05:12.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:12 smithi153 bash[27868]: cluster 2023-10-09T04:05:11.507677+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:05:13.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:13 smithi082 bash[27462]: cluster 2023-10-09T04:05:12.546039+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:13.897 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:13 smithi082 bash[33901]: cluster 2023-10-09T04:05:12.546039+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:13.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:13 smithi153 bash[27868]: cluster 2023-10-09T04:05:12.546039+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:15.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:14 smithi082 bash[33901]: cluster 2023-10-09T04:05:14.546728+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:15.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:14 smithi082 bash[27462]: cluster 2023-10-09T04:05:14.546728+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:15.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:14 smithi153 bash[27868]: cluster 2023-10-09T04:05:14.546728+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:16.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[27462]: audit 2023-10-09T04:05:16.434659+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:16.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[27462]: audit 2023-10-09T04:05:16.437336+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:16.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[27462]: audit 2023-10-09T04:05:16.438200+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:16.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[33901]: audit 2023-10-09T04:05:16.434659+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:16.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[33901]: audit 2023-10-09T04:05:16.437336+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:16.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:16 smithi082 bash[33901]: audit 2023-10-09T04:05:16.438200+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:16.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:16 smithi153 bash[27868]: audit 2023-10-09T04:05:16.434659+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:16.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:16 smithi153 bash[27868]: audit 2023-10-09T04:05:16.437336+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:16.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:16 smithi153 bash[27868]: audit 2023-10-09T04:05:16.438200+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:17.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:17 smithi153 bash[27868]: audit 2023-10-09T04:05:16.431679+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:17.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:17 smithi153 bash[27868]: cluster 2023-10-09T04:05:16.547404+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:17.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:17 smithi082 bash[27462]: audit 2023-10-09T04:05:16.431679+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:17 smithi082 bash[27462]: cluster 2023-10-09T04:05:16.547404+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:17 smithi082 bash[33901]: audit 2023-10-09T04:05:16.431679+0000 mgr.y (mgr.14146) 131 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:17 smithi082 bash[33901]: cluster 2023-10-09T04:05:16.547404+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: cephadm 2023-10-09T04:05:17.596608+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.605253+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.606902+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.608108+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.609239+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: cephadm 2023-10-09T04:05:17.610200+0000 mgr.y (mgr.14146) 134 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 5413M 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.615591+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.617118+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.618127+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:18 smithi153 bash[27868]: audit 2023-10-09T04:05:17.626012+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: cephadm 2023-10-09T04:05:17.596608+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.605253+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.606902+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.608108+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.609239+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: cephadm 2023-10-09T04:05:17.610200+0000 mgr.y (mgr.14146) 134 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 5413M 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.615591+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.617118+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.618127+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[27462]: audit 2023-10-09T04:05:17.626012+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: cephadm 2023-10-09T04:05:17.596608+0000 mgr.y (mgr.14146) 133 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.605253+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.606902+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.608108+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.609239+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: cephadm 2023-10-09T04:05:17.610200+0000 mgr.y (mgr.14146) 134 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 5413M 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.615591+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.617118+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.618127+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:18.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:18 smithi082 bash[33901]: audit 2023-10-09T04:05:17.626012+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:19.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:19 smithi153 bash[27868]: cluster 2023-10-09T04:05:18.547961+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:19.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:19 smithi082 bash[27462]: cluster 2023-10-09T04:05:18.547961+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:19 smithi082 bash[33901]: cluster 2023-10-09T04:05:18.547961+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:21 smithi153 bash[27868]: cluster 2023-10-09T04:05:20.548583+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:21.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:21 smithi082 bash[27462]: cluster 2023-10-09T04:05:20.548583+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:21.938 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:21 smithi082 bash[33901]: cluster 2023-10-09T04:05:20.548583+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:23.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:23 smithi082 bash[27462]: cluster 2023-10-09T04:05:22.548952+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:23.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:23 smithi082 bash[33901]: cluster 2023-10-09T04:05:22.548952+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:23.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:23 smithi153 bash[27868]: cluster 2023-10-09T04:05:22.548952+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[27462]: audit 2023-10-09T04:05:24.390516+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]: dispatch 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[27462]: audit 2023-10-09T04:05:24.402942+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]': finished 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[27462]: cluster 2023-10-09T04:05:24.403034+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[27462]: audit 2023-10-09T04:05:24.403173+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[27462]: cluster 2023-10-09T04:05:24.549464+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[33901]: audit 2023-10-09T04:05:24.390516+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]: dispatch 2023-10-09T04:05:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[33901]: audit 2023-10-09T04:05:24.402942+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]': finished 2023-10-09T04:05:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[33901]: cluster 2023-10-09T04:05:24.403034+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:05:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[33901]: audit 2023-10-09T04:05:24.403173+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:24 smithi082 bash[33901]: cluster 2023-10-09T04:05:24.549464+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:25.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:24 smithi153 bash[27868]: audit 2023-10-09T04:05:24.390516+0000 mon.a (mon.0) 339 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]: dispatch 2023-10-09T04:05:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:24 smithi153 bash[27868]: audit 2023-10-09T04:05:24.402942+0000 mon.a (mon.0) 340 : audit [INF] from='client.? 172.21.15.82:0/3050276840' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ae42ef8-7a8b-4eec-927b-01af15ae54f1"}]': finished 2023-10-09T04:05:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:24 smithi153 bash[27868]: cluster 2023-10-09T04:05:24.403034+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:05:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:24 smithi153 bash[27868]: audit 2023-10-09T04:05:24.403173+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:24 smithi153 bash[27868]: cluster 2023-10-09T04:05:24.549464+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:26.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:25 smithi082 bash[27462]: audit 2023-10-09T04:05:25.111042+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.82:0/4141271363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:26.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:25 smithi082 bash[33901]: audit 2023-10-09T04:05:25.111042+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.82:0/4141271363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:26.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:25 smithi153 bash[27868]: audit 2023-10-09T04:05:25.111042+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.82:0/4141271363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:27.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:26 smithi082 bash[27462]: cluster 2023-10-09T04:05:26.549962+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:26 smithi082 bash[33901]: cluster 2023-10-09T04:05:26.549962+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:26 smithi153 bash[27868]: cluster 2023-10-09T04:05:26.549962+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:29 smithi153 bash[27868]: cluster 2023-10-09T04:05:28.550494+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:29 smithi082 bash[27462]: cluster 2023-10-09T04:05:28.550494+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:29 smithi082 bash[33901]: cluster 2023-10-09T04:05:28.550494+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:31.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:31 smithi153 bash[27868]: cluster 2023-10-09T04:05:30.551112+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:31 smithi082 bash[27462]: cluster 2023-10-09T04:05:30.551112+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:31.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:31 smithi082 bash[33901]: cluster 2023-10-09T04:05:30.551112+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:33.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:33 smithi153 bash[27868]: cluster 2023-10-09T04:05:32.551721+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:33.925 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:33 smithi082 bash[33901]: cluster 2023-10-09T04:05:32.551721+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:33.925 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:33 smithi082 bash[27462]: cluster 2023-10-09T04:05:32.551721+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:35.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:34 smithi082 bash[33901]: cluster 2023-10-09T04:05:34.552367+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:35.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:34 smithi082 bash[27462]: cluster 2023-10-09T04:05:34.552367+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:35.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:34 smithi153 bash[27868]: cluster 2023-10-09T04:05:34.552367+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:37.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:37 smithi153 bash[27868]: cluster 2023-10-09T04:05:36.552840+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:37.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:37 smithi153 bash[27868]: audit 2023-10-09T04:05:36.766537+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:05:37.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:37 smithi153 bash[27868]: audit 2023-10-09T04:05:36.767904+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:37.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:37 smithi153 bash[27868]: cephadm 2023-10-09T04:05:36.768968+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Deploying daemon osd.3 on smithi082 2023-10-09T04:05:37.966 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[33901]: cluster 2023-10-09T04:05:36.552840+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:37.966 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[33901]: audit 2023-10-09T04:05:36.766537+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:05:37.966 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[33901]: audit 2023-10-09T04:05:36.767904+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:37.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[33901]: cephadm 2023-10-09T04:05:36.768968+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Deploying daemon osd.3 on smithi082 2023-10-09T04:05:37.967 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[27462]: cluster 2023-10-09T04:05:36.552840+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:37.967 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[27462]: audit 2023-10-09T04:05:36.766537+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:05:37.967 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[27462]: audit 2023-10-09T04:05:36.767904+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:37.967 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:37 smithi082 bash[27462]: cephadm 2023-10-09T04:05:36.768968+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Deploying daemon osd.3 on smithi082 2023-10-09T04:05:39.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:39 smithi153 bash[27868]: cluster 2023-10-09T04:05:38.553553+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:39.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:39 smithi082 bash[33901]: cluster 2023-10-09T04:05:38.553553+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:39 smithi082 bash[27462]: cluster 2023-10-09T04:05:38.553553+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[33901]: cluster 2023-10-09T04:05:40.554071+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[33901]: audit 2023-10-09T04:05:40.696345+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[33901]: audit 2023-10-09T04:05:40.703619+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[27462]: cluster 2023-10-09T04:05:40.554071+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[27462]: audit 2023-10-09T04:05:40.696345+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:41.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:41 smithi082 bash[27462]: audit 2023-10-09T04:05:40.703619+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:42.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:41 smithi153 bash[27868]: cluster 2023-10-09T04:05:40.554071+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:41 smithi153 bash[27868]: audit 2023-10-09T04:05:40.696345+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:41 smithi153 bash[27868]: audit 2023-10-09T04:05:40.703619+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:43 smithi082 bash[27462]: cluster 2023-10-09T04:05:42.554697+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:43.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:43 smithi082 bash[33901]: cluster 2023-10-09T04:05:42.554697+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:44.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:43 smithi153 bash[27868]: cluster 2023-10-09T04:05:42.554697+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:45.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:44 smithi082 bash[27462]: cluster 2023-10-09T04:05:44.555233+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:45.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:44 smithi082 bash[33901]: cluster 2023-10-09T04:05:44.555233+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:45.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:44 smithi153 bash[27868]: cluster 2023-10-09T04:05:44.555233+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:47.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[27462]: audit 2023-10-09T04:05:45.930330+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[27462]: audit 2023-10-09T04:05:45.931616+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[27462]: audit 2023-10-09T04:05:45.933266+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[27462]: audit 2023-10-09T04:05:45.939151+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[33901]: audit 2023-10-09T04:05:45.930330+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[33901]: audit 2023-10-09T04:05:45.931616+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[33901]: audit 2023-10-09T04:05:45.933266+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:47.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:46 smithi082 bash[33901]: audit 2023-10-09T04:05:45.939151+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:47.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:46 smithi153 bash[27868]: audit 2023-10-09T04:05:45.930330+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:46 smithi153 bash[27868]: audit 2023-10-09T04:05:45.931616+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:46 smithi153 bash[27868]: audit 2023-10-09T04:05:45.933266+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:46 smithi153 bash[27868]: audit 2023-10-09T04:05:45.939151+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:48.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:47 smithi082 bash[27462]: cluster 2023-10-09T04:05:46.555711+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:48.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:47 smithi082 bash[27462]: audit 2023-10-09T04:05:47.349152+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:05:48.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:47 smithi082 bash[33901]: cluster 2023-10-09T04:05:46.555711+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:48.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:47 smithi082 bash[33901]: audit 2023-10-09T04:05:47.349152+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:05:48.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:47 smithi153 bash[27868]: cluster 2023-10-09T04:05:46.555711+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:47 smithi153 bash[27868]: audit 2023-10-09T04:05:47.349152+0000 mon.a (mon.0) 351 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[27462]: audit 2023-10-09T04:05:47.945113+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[27462]: cluster 2023-10-09T04:05:47.945230+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[27462]: audit 2023-10-09T04:05:47.945429+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[27462]: audit 2023-10-09T04:05:47.946172+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[27462]: cluster 2023-10-09T04:05:48.556190+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:49.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[33901]: audit 2023-10-09T04:05:47.945113+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:05:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[33901]: cluster 2023-10-09T04:05:47.945230+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:05:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[33901]: audit 2023-10-09T04:05:47.945429+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[33901]: audit 2023-10-09T04:05:47.946172+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:48 smithi082 bash[33901]: cluster 2023-10-09T04:05:48.556190+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:48 smithi153 bash[27868]: audit 2023-10-09T04:05:47.945113+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:05:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:48 smithi153 bash[27868]: cluster 2023-10-09T04:05:47.945230+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:05:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:48 smithi153 bash[27868]: audit 2023-10-09T04:05:47.945429+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:48 smithi153 bash[27868]: audit 2023-10-09T04:05:47.946172+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:05:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:48 smithi153 bash[27868]: cluster 2023-10-09T04:05:48.556190+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:49.491 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 3 on host 'smithi082' 2023-10-09T04:05:50.158 DEBUG:teuthology.orchestra.run.smithi082:osd.3> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T04:05:50.161 INFO:tasks.cephadm:Deploying osd.4 on smithi153 with /dev/vg_nvme/lv_4... 2023-10-09T04:05:50.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:05:50.209 INFO:journalctl@ceph.osd.3.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:05:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: audit 2023-10-09T04:05:48.949318+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: cluster 2023-10-09T04:05:48.949500+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:05:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: audit 2023-10-09T04:05:48.950145+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: audit 2023-10-09T04:05:48.957377+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: audit 2023-10-09T04:05:49.489521+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[27462]: audit 2023-10-09T04:05:49.514973+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: audit 2023-10-09T04:05:48.949318+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: cluster 2023-10-09T04:05:48.949500+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: audit 2023-10-09T04:05:48.950145+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: audit 2023-10-09T04:05:48.957377+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: audit 2023-10-09T04:05:49.489521+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:49 smithi082 bash[33901]: audit 2023-10-09T04:05:49.514973+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: audit 2023-10-09T04:05:48.949318+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: cluster 2023-10-09T04:05:48.949500+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: audit 2023-10-09T04:05:48.950145+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: audit 2023-10-09T04:05:48.957377+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: audit 2023-10-09T04:05:49.489521+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:50.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:49 smithi153 bash[27868]: audit 2023-10-09T04:05:49.514973+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: cluster 2023-10-09T04:05:48.329254+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: cluster 2023-10-09T04:05:48.329407+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: audit 2023-10-09T04:05:49.960249+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: cluster 2023-10-09T04:05:49.964880+0000 mon.a (mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210] boot 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: cluster 2023-10-09T04:05:49.964923+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:05:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: audit 2023-10-09T04:05:49.965491+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[27462]: cluster 2023-10-09T04:05:50.556731+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: cluster 2023-10-09T04:05:48.329254+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: cluster 2023-10-09T04:05:48.329407+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: audit 2023-10-09T04:05:49.960249+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: cluster 2023-10-09T04:05:49.964880+0000 mon.a (mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210] boot 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: cluster 2023-10-09T04:05:49.964923+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: audit 2023-10-09T04:05:49.965491+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:50 smithi082 bash[33901]: cluster 2023-10-09T04:05:50.556731+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: cluster 2023-10-09T04:05:48.329254+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: cluster 2023-10-09T04:05:48.329407+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: audit 2023-10-09T04:05:49.960249+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: cluster 2023-10-09T04:05:49.964880+0000 mon.a (mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.82:6826/3309313210,v1:172.21.15.82:6827/3309313210] boot 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: cluster 2023-10-09T04:05:49.964923+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: audit 2023-10-09T04:05:49.965491+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:05:51.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:50 smithi153 bash[27868]: cluster 2023-10-09T04:05:50.556731+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:05:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:51 smithi082 bash[27462]: cluster 2023-10-09T04:05:50.972089+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:05:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:51 smithi082 bash[33901]: cluster 2023-10-09T04:05:50.972089+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:51 smithi153 bash[27868]: cluster 2023-10-09T04:05:50.972089+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:05:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:52 smithi082 bash[33901]: cluster 2023-10-09T04:05:52.557321+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:52 smithi082 bash[27462]: cluster 2023-10-09T04:05:52.557321+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0177187 s, 592 MB/s 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 2023-10-09T04:05:53.383 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2023-10-09T04:05:53.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:52 smithi153 bash[27868]: cluster 2023-10-09T04:05:52.557321+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:54.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi153:vg_nvme/lv_4 2023-10-09T04:05:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:54 smithi082 bash[27462]: cluster 2023-10-09T04:05:54.557820+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:55.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:54 smithi082 bash[33901]: cluster 2023-10-09T04:05:54.557820+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:55.334 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:54 smithi153 bash[27868]: cluster 2023-10-09T04:05:54.557820+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[27462]: cluster 2023-10-09T04:05:56.558349+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[27462]: audit 2023-10-09T04:05:56.834440+0000 mgr.y (mgr.14146) 156 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[27462]: audit 2023-10-09T04:05:56.841662+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[27462]: audit 2023-10-09T04:05:56.846097+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[27462]: audit 2023-10-09T04:05:56.847380+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[33901]: cluster 2023-10-09T04:05:56.558349+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[33901]: audit 2023-10-09T04:05:56.834440+0000 mgr.y (mgr.14146) 156 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[33901]: audit 2023-10-09T04:05:56.841662+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[33901]: audit 2023-10-09T04:05:56.846097+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:57 smithi082 bash[33901]: audit 2023-10-09T04:05:56.847380+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:57 smithi153 bash[27868]: cluster 2023-10-09T04:05:56.558349+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:57 smithi153 bash[27868]: audit 2023-10-09T04:05:56.834440+0000 mgr.y (mgr.14146) 156 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:57 smithi153 bash[27868]: audit 2023-10-09T04:05:56.841662+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:57 smithi153 bash[27868]: audit 2023-10-09T04:05:56.846097+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:57 smithi153 bash[27868]: audit 2023-10-09T04:05:56.847380+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: cephadm 2023-10-09T04:05:58.298728+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.308191+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.309821+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.311177+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.312401+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.313643+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: cephadm 2023-10-09T04:05:58.314634+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 4060M 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.320684+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.340175+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.341952+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: audit 2023-10-09T04:05:58.349976+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:05:59 smithi153 bash[27868]: cluster 2023-10-09T04:05:58.558969+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: cephadm 2023-10-09T04:05:58.298728+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.308191+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.309821+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.311177+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.312401+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.313643+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: cephadm 2023-10-09T04:05:58.314634+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 4060M 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.320684+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.340175+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.341952+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: audit 2023-10-09T04:05:58.349976+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[27462]: cluster 2023-10-09T04:05:58.558969+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:05:59.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: cephadm 2023-10-09T04:05:58.298728+0000 mgr.y (mgr.14146) 157 : cephadm [INF] Detected new or changed devices on smithi082 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.308191+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.309821+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.311177+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.312401+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.313643+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: cephadm 2023-10-09T04:05:58.314634+0000 mgr.y (mgr.14146) 158 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 4060M 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.320684+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.340175+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.341952+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: audit 2023-10-09T04:05:58.349976+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:05:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:05:59 smithi082 bash[33901]: cluster 2023-10-09T04:05:58.558969+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:01.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:01 smithi153 bash[27868]: cluster 2023-10-09T04:06:00.559466+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:01.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:01 smithi082 bash[27462]: cluster 2023-10-09T04:06:00.559466+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:01 smithi082 bash[33901]: cluster 2023-10-09T04:06:00.559466+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:03.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:03 smithi153 bash[27868]: cluster 2023-10-09T04:06:02.560031+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:03 smithi082 bash[27462]: cluster 2023-10-09T04:06:02.560031+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:03 smithi082 bash[33901]: cluster 2023-10-09T04:06:02.560031+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: cluster 2023-10-09T04:06:04.560539+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: audit 2023-10-09T04:06:04.886885+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.153:0/3816156620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: audit 2023-10-09T04:06:04.887589+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: audit 2023-10-09T04:06:04.899477+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]': finished 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: cluster 2023-10-09T04:06:04.899663+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:06:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[27462]: audit 2023-10-09T04:06:04.899973+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: cluster 2023-10-09T04:06:04.560539+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: audit 2023-10-09T04:06:04.886885+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.153:0/3816156620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: audit 2023-10-09T04:06:04.887589+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: audit 2023-10-09T04:06:04.899477+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]': finished 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: cluster 2023-10-09T04:06:04.899663+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:06:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:04 smithi082 bash[33901]: audit 2023-10-09T04:06:04.899973+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: cluster 2023-10-09T04:06:04.560539+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: audit 2023-10-09T04:06:04.886885+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.153:0/3816156620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: audit 2023-10-09T04:06:04.887589+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]: dispatch 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: audit 2023-10-09T04:06:04.899477+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "298364f7-1ee4-436d-81d7-1ddf54d45607"}]': finished 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: cluster 2023-10-09T04:06:04.899663+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:06:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:04 smithi153 bash[27868]: audit 2023-10-09T04:06:04.899973+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:05 smithi082 bash[27462]: audit 2023-10-09T04:06:05.611169+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.153:0/4072460340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:06.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:05 smithi082 bash[33901]: audit 2023-10-09T04:06:05.611169+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.153:0/4072460340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:05 smithi153 bash[27868]: audit 2023-10-09T04:06:05.611169+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.153:0/4072460340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:07.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:06 smithi153 bash[27868]: cluster 2023-10-09T04:06:06.561000+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:06 smithi082 bash[27462]: cluster 2023-10-09T04:06:06.561000+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:07.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:06 smithi082 bash[33901]: cluster 2023-10-09T04:06:06.561000+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:09.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:09 smithi153 bash[27868]: cluster 2023-10-09T04:06:08.561540+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:09.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:09 smithi082 bash[27462]: cluster 2023-10-09T04:06:08.561540+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:09.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:09 smithi082 bash[33901]: cluster 2023-10-09T04:06:08.561540+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:11.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:11 smithi153 bash[27868]: cluster 2023-10-09T04:06:10.562153+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:11 smithi082 bash[27462]: cluster 2023-10-09T04:06:10.562153+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:11.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:11 smithi082 bash[33901]: cluster 2023-10-09T04:06:10.562153+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:13.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:13 smithi153 bash[27868]: cluster 2023-10-09T04:06:12.562774+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:13.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:13 smithi082 bash[27462]: cluster 2023-10-09T04:06:12.562774+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:13.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:13 smithi082 bash[33901]: cluster 2023-10-09T04:06:12.562774+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:15.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:14 smithi153 bash[27868]: cluster 2023-10-09T04:06:14.563402+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:15.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:14 smithi082 bash[27462]: cluster 2023-10-09T04:06:14.563402+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:15.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:14 smithi082 bash[33901]: cluster 2023-10-09T04:06:14.563402+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:17 smithi153 bash[27868]: cluster 2023-10-09T04:06:16.563954+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:17 smithi153 bash[27868]: audit 2023-10-09T04:06:17.333850+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:06:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:17 smithi153 bash[27868]: audit 2023-10-09T04:06:17.335709+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[27462]: cluster 2023-10-09T04:06:16.563954+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[27462]: audit 2023-10-09T04:06:17.333850+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[27462]: audit 2023-10-09T04:06:17.335709+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[33901]: cluster 2023-10-09T04:06:16.563954+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[33901]: audit 2023-10-09T04:06:17.333850+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:06:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:17 smithi082 bash[33901]: audit 2023-10-09T04:06:17.335709+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:18.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:18 smithi153 bash[27868]: cephadm 2023-10-09T04:06:17.337333+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Deploying daemon osd.4 on smithi153 2023-10-09T04:06:18.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:18 smithi082 bash[27462]: cephadm 2023-10-09T04:06:17.337333+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Deploying daemon osd.4 on smithi153 2023-10-09T04:06:18.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:18 smithi082 bash[33901]: cephadm 2023-10-09T04:06:17.337333+0000 mgr.y (mgr.14146) 169 : cephadm [INF] Deploying daemon osd.4 on smithi153 2023-10-09T04:06:19.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:19 smithi153 bash[27868]: cluster 2023-10-09T04:06:18.564514+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:19.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:19 smithi082 bash[27462]: cluster 2023-10-09T04:06:18.564514+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:19 smithi082 bash[33901]: cluster 2023-10-09T04:06:18.564514+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:21 smithi153 bash[27868]: cluster 2023-10-09T04:06:20.565097+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:21 smithi153 bash[27868]: audit 2023-10-09T04:06:21.385459+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:21 smithi153 bash[27868]: audit 2023-10-09T04:06:21.385886+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[27462]: cluster 2023-10-09T04:06:20.565097+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[27462]: audit 2023-10-09T04:06:21.385459+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[27462]: audit 2023-10-09T04:06:21.385886+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[33901]: cluster 2023-10-09T04:06:20.565097+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[33901]: audit 2023-10-09T04:06:21.385459+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:21 smithi082 bash[33901]: audit 2023-10-09T04:06:21.385886+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:23.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:23 smithi153 bash[27868]: cluster 2023-10-09T04:06:22.565667+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:23.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:23 smithi082 bash[27462]: cluster 2023-10-09T04:06:22.565667+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:23.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:23 smithi082 bash[33901]: cluster 2023-10-09T04:06:22.565667+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:24 smithi082 bash[27462]: cluster 2023-10-09T04:06:24.566220+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:25.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:24 smithi082 bash[33901]: cluster 2023-10-09T04:06:24.566220+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:24 smithi153 bash[27868]: cluster 2023-10-09T04:06:24.566220+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:27 smithi153 bash[27868]: audit 2023-10-09T04:06:26.483022+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:27 smithi153 bash[27868]: cluster 2023-10-09T04:06:26.566798+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:27 smithi153 bash[27868]: audit 2023-10-09T04:06:26.757094+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:27.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:27 smithi153 bash[27868]: audit 2023-10-09T04:06:26.759228+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:27.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:27 smithi153 bash[27868]: audit 2023-10-09T04:06:26.767661+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:27.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[27462]: audit 2023-10-09T04:06:26.483022+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[27462]: cluster 2023-10-09T04:06:26.566798+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[27462]: audit 2023-10-09T04:06:26.757094+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[27462]: audit 2023-10-09T04:06:26.759228+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[27462]: audit 2023-10-09T04:06:26.767661+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[33901]: audit 2023-10-09T04:06:26.483022+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[33901]: cluster 2023-10-09T04:06:26.566798+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[33901]: audit 2023-10-09T04:06:26.757094+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:27.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[33901]: audit 2023-10-09T04:06:26.759228+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:27.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:27 smithi082 bash[33901]: audit 2023-10-09T04:06:26.767661+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:28.769 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:28 smithi153 bash[27868]: audit 2023-10-09T04:06:27.963134+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:28.769 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:28 smithi153 bash[27868]: audit 2023-10-09T04:06:27.963858+0000 mon.a (mon.0) 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:28 smithi082 bash[27462]: audit 2023-10-09T04:06:27.963134+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:28 smithi082 bash[27462]: audit 2023-10-09T04:06:27.963858+0000 mon.a (mon.0) 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:28.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:28 smithi082 bash[33901]: audit 2023-10-09T04:06:27.963134+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:28.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:28 smithi082 bash[33901]: audit 2023-10-09T04:06:27.963858+0000 mon.a (mon.0) 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: audit 2023-10-09T04:06:28.504487+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: cluster 2023-10-09T04:06:28.504615+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: audit 2023-10-09T04:06:28.504846+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: audit 2023-10-09T04:06:28.505677+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: audit 2023-10-09T04:06:28.506263+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:29 smithi153 bash[27868]: cluster 2023-10-09T04:06:28.567331+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: audit 2023-10-09T04:06:28.504487+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:06:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: cluster 2023-10-09T04:06:28.504615+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:06:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: audit 2023-10-09T04:06:28.504846+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: audit 2023-10-09T04:06:28.505677+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: audit 2023-10-09T04:06:28.506263+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[27462]: cluster 2023-10-09T04:06:28.567331+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: audit 2023-10-09T04:06:28.504487+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: cluster 2023-10-09T04:06:28.504615+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: audit 2023-10-09T04:06:28.504846+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: audit 2023-10-09T04:06:28.505677+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: audit 2023-10-09T04:06:28.506263+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:06:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:29 smithi082 bash[33901]: cluster 2023-10-09T04:06:28.567331+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:30.016 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 4 on host 'smithi153' 2023-10-09T04:06:30.620 DEBUG:teuthology.orchestra.run.smithi153:osd.4> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T04:06:30.622 INFO:tasks.cephadm:Deploying osd.5 on smithi153 with /dev/vg_nvme/lv_3... 2023-10-09T04:06:30.622 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:06:30.830 INFO:journalctl@ceph.osd.4.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:06:30.863 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: audit 2023-10-09T04:06:29.510301+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:06:30.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: cluster 2023-10-09T04:06:29.510429+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:06:30.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: audit 2023-10-09T04:06:29.511191+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: audit 2023-10-09T04:06:29.517941+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: audit 2023-10-09T04:06:30.013881+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:30.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:30 smithi153 bash[27868]: audit 2023-10-09T04:06:30.045326+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: audit 2023-10-09T04:06:29.510301+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:06:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: cluster 2023-10-09T04:06:29.510429+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:06:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: audit 2023-10-09T04:06:29.511191+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: audit 2023-10-09T04:06:29.517941+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: audit 2023-10-09T04:06:30.013881+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[27462]: audit 2023-10-09T04:06:30.045326+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: audit 2023-10-09T04:06:29.510301+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: cluster 2023-10-09T04:06:29.510429+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: audit 2023-10-09T04:06:29.511191+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: audit 2023-10-09T04:06:29.517941+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: audit 2023-10-09T04:06:30.013881+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:30 smithi082 bash[33901]: audit 2023-10-09T04:06:30.045326+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:28.997321+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:28.997403+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:30.520500+0000 mon.a (mon.0) 402 : cluster [INF] osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063] boot 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:30.520586+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: audit 2023-10-09T04:06:30.521379+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:31.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:30.567902+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:31.800 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:31 smithi153 bash[27868]: cluster 2023-10-09T04:06:31.525453+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:06:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:28.997321+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:06:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:28.997403+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:06:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:30.520500+0000 mon.a (mon.0) 402 : cluster [INF] osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063] boot 2023-10-09T04:06:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:30.520586+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:06:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: audit 2023-10-09T04:06:30.521379+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:30.567902+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[27462]: cluster 2023-10-09T04:06:31.525453+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:28.997321+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:28.997403+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:30.520500+0000 mon.a (mon.0) 402 : cluster [INF] osd.4 [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063] boot 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:30.520586+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: audit 2023-10-09T04:06:30.521379+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:30.567902+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:06:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:31 smithi082 bash[33901]: cluster 2023-10-09T04:06:31.525453+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0172688 s, 607 MB/s 2023-10-09T04:06:33.793 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2023-10-09T04:06:33.859 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:33 smithi153 bash[27868]: cluster 2023-10-09T04:06:32.529992+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:06:33.859 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:33 smithi153 bash[27868]: cluster 2023-10-09T04:06:32.568428+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:33 smithi082 bash[27462]: cluster 2023-10-09T04:06:32.529992+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:06:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:33 smithi082 bash[27462]: cluster 2023-10-09T04:06:32.568428+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:33.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:33 smithi082 bash[33901]: cluster 2023-10-09T04:06:32.529992+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:06:33.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:33 smithi082 bash[33901]: cluster 2023-10-09T04:06:32.568428+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:34.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi153:vg_nvme/lv_3 2023-10-09T04:06:35.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:34 smithi082 bash[33901]: cluster 2023-10-09T04:06:34.568934+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:35.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:34 smithi082 bash[27462]: cluster 2023-10-09T04:06:34.568934+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:35.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:34 smithi153 bash[27868]: cluster 2023-10-09T04:06:34.568934+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:37.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:37 smithi153 bash[27868]: cluster 2023-10-09T04:06:36.569490+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:37 smithi082 bash[27462]: cluster 2023-10-09T04:06:36.569490+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:37 smithi082 bash[33901]: cluster 2023-10-09T04:06:36.569490+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:39 smithi153 bash[27868]: cluster 2023-10-09T04:06:38.569976+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:39 smithi153 bash[27868]: audit 2023-10-09T04:06:38.907442+0000 mgr.y (mgr.14146) 181 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:39 smithi153 bash[27868]: audit 2023-10-09T04:06:38.914189+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:06:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:39 smithi153 bash[27868]: audit 2023-10-09T04:06:38.918617+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:06:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:39 smithi153 bash[27868]: audit 2023-10-09T04:06:38.919956+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[27462]: cluster 2023-10-09T04:06:38.569976+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[27462]: audit 2023-10-09T04:06:38.907442+0000 mgr.y (mgr.14146) 181 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:39.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[27462]: audit 2023-10-09T04:06:38.914189+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:06:39.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[27462]: audit 2023-10-09T04:06:38.918617+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:06:39.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[27462]: audit 2023-10-09T04:06:38.919956+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[33901]: cluster 2023-10-09T04:06:38.569976+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:39.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[33901]: audit 2023-10-09T04:06:38.907442+0000 mgr.y (mgr.14146) 181 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:39.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[33901]: audit 2023-10-09T04:06:38.914189+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:06:39.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[33901]: audit 2023-10-09T04:06:38.918617+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:06:39.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:39 smithi082 bash[33901]: audit 2023-10-09T04:06:38.919956+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: cephadm 2023-10-09T04:06:40.054215+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.063313+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.064780+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: cephadm 2023-10-09T04:06:40.065731+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17265M 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.072395+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.073891+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.075300+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: audit 2023-10-09T04:06:40.083407+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:41 smithi153 bash[27868]: cluster 2023-10-09T04:06:40.570523+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: cephadm 2023-10-09T04:06:40.054215+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:06:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.063313+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.064780+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: cephadm 2023-10-09T04:06:40.065731+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17265M 2023-10-09T04:06:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.072395+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.073891+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.075300+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: audit 2023-10-09T04:06:40.083407+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[27462]: cluster 2023-10-09T04:06:40.570523+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: cephadm 2023-10-09T04:06:40.054215+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.063313+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.064780+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: cephadm 2023-10-09T04:06:40.065731+0000 mgr.y (mgr.14146) 183 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 17265M 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.072395+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.073891+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.075300+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: audit 2023-10-09T04:06:40.083407+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:06:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:41 smithi082 bash[33901]: cluster 2023-10-09T04:06:40.570523+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:43.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:43 smithi153 bash[27868]: cluster 2023-10-09T04:06:42.571075+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:43 smithi082 bash[27462]: cluster 2023-10-09T04:06:42.571075+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:43.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:43 smithi082 bash[33901]: cluster 2023-10-09T04:06:42.571075+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:44 smithi153 bash[27868]: cluster 2023-10-09T04:06:44.571619+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:44 smithi082 bash[27462]: cluster 2023-10-09T04:06:44.571619+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:45.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:44 smithi082 bash[33901]: cluster 2023-10-09T04:06:44.571619+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: cluster 2023-10-09T04:06:46.572154+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: audit 2023-10-09T04:06:47.013596+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.153:0/1186802090' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: audit 2023-10-09T04:06:47.014617+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: audit 2023-10-09T04:06:47.022700+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]': finished 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: cluster 2023-10-09T04:06:47.022823+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T04:06:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:47 smithi153 bash[27868]: audit 2023-10-09T04:06:47.023058+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:47.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: cluster 2023-10-09T04:06:46.572154+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:47.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: audit 2023-10-09T04:06:47.013596+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.153:0/1186802090' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: audit 2023-10-09T04:06:47.014617+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: audit 2023-10-09T04:06:47.022700+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]': finished 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: cluster 2023-10-09T04:06:47.022823+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[27462]: audit 2023-10-09T04:06:47.023058+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: cluster 2023-10-09T04:06:46.572154+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: audit 2023-10-09T04:06:47.013596+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.153:0/1186802090' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: audit 2023-10-09T04:06:47.014617+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]: dispatch 2023-10-09T04:06:47.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: audit 2023-10-09T04:06:47.022700+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d79acac-f278-473a-8dbf-9cf495c9f498"}]': finished 2023-10-09T04:06:47.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: cluster 2023-10-09T04:06:47.022823+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T04:06:47.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:47 smithi082 bash[33901]: audit 2023-10-09T04:06:47.023058+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:48.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:48 smithi153 bash[27868]: audit 2023-10-09T04:06:47.766559+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.153:0/3149449472' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:48.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:48 smithi082 bash[33901]: audit 2023-10-09T04:06:47.766559+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.153:0/3149449472' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:48.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:48 smithi082 bash[27462]: audit 2023-10-09T04:06:47.766559+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.153:0/3149449472' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:06:49.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:49 smithi153 bash[27868]: cluster 2023-10-09T04:06:48.572678+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:49 smithi082 bash[27462]: cluster 2023-10-09T04:06:48.572678+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:49.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:49 smithi082 bash[33901]: cluster 2023-10-09T04:06:48.572678+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:51.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:51 smithi153 bash[27868]: cluster 2023-10-09T04:06:50.573186+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:51.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:51 smithi082 bash[27462]: cluster 2023-10-09T04:06:50.573186+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:51 smithi082 bash[33901]: cluster 2023-10-09T04:06:50.573186+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:53 smithi153 bash[27868]: cluster 2023-10-09T04:06:52.573780+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:53 smithi082 bash[27462]: cluster 2023-10-09T04:06:52.573780+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:53.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:53 smithi082 bash[33901]: cluster 2023-10-09T04:06:52.573780+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:54 smithi082 bash[27462]: cluster 2023-10-09T04:06:54.574294+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:55.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:54 smithi082 bash[33901]: cluster 2023-10-09T04:06:54.574294+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:54 smithi153 bash[27868]: cluster 2023-10-09T04:06:54.574294+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:57 smithi153 bash[27868]: cluster 2023-10-09T04:06:56.574834+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:57 smithi082 bash[33901]: cluster 2023-10-09T04:06:56.574834+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:57.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:57 smithi082 bash[27462]: cluster 2023-10-09T04:06:56.574834+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:59.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:59 smithi153 bash[27868]: cluster 2023-10-09T04:06:58.575362+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:59.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:59 smithi153 bash[27868]: audit 2023-10-09T04:06:59.208909+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:06:59.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:06:59 smithi153 bash[27868]: audit 2023-10-09T04:06:59.210538+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[27462]: cluster 2023-10-09T04:06:58.575362+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[27462]: audit 2023-10-09T04:06:59.208909+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:06:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[27462]: audit 2023-10-09T04:06:59.210538+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:59.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[33901]: cluster 2023-10-09T04:06:58.575362+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:06:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[33901]: audit 2023-10-09T04:06:59.208909+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:06:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:06:59 smithi082 bash[33901]: audit 2023-10-09T04:06:59.210538+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:00.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:00 smithi153 bash[27868]: cephadm 2023-10-09T04:06:59.211941+0000 mgr.y (mgr.14146) 194 : cephadm [INF] Deploying daemon osd.5 on smithi153 2023-10-09T04:07:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:00 smithi082 bash[27462]: cephadm 2023-10-09T04:06:59.211941+0000 mgr.y (mgr.14146) 194 : cephadm [INF] Deploying daemon osd.5 on smithi153 2023-10-09T04:07:00.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:00 smithi082 bash[33901]: cephadm 2023-10-09T04:06:59.211941+0000 mgr.y (mgr.14146) 194 : cephadm [INF] Deploying daemon osd.5 on smithi153 2023-10-09T04:07:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:01 smithi153 bash[27868]: cluster 2023-10-09T04:07:00.575881+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:01 smithi082 bash[33901]: cluster 2023-10-09T04:07:00.575881+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:01.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:01 smithi082 bash[27462]: cluster 2023-10-09T04:07:00.575881+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:03.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:03 smithi153 bash[27868]: cluster 2023-10-09T04:07:02.576518+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:03.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:03 smithi153 bash[27868]: audit 2023-10-09T04:07:03.312049+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:03.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:03 smithi153 bash[27868]: audit 2023-10-09T04:07:03.321169+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[27462]: cluster 2023-10-09T04:07:02.576518+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[27462]: audit 2023-10-09T04:07:03.312049+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[27462]: audit 2023-10-09T04:07:03.321169+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[33901]: cluster 2023-10-09T04:07:02.576518+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[33901]: audit 2023-10-09T04:07:03.312049+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:03 smithi082 bash[33901]: audit 2023-10-09T04:07:03.321169+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:04 smithi153 bash[27868]: cluster 2023-10-09T04:07:04.577113+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:05.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:04 smithi082 bash[33901]: cluster 2023-10-09T04:07:04.577113+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:05.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:04 smithi082 bash[27462]: cluster 2023-10-09T04:07:04.577113+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:07 smithi153 bash[27868]: cluster 2023-10-09T04:07:06.577626+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:07.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:07 smithi082 bash[27462]: cluster 2023-10-09T04:07:06.577626+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:07.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:07 smithi082 bash[33901]: cluster 2023-10-09T04:07:06.577626+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:09 smithi153 bash[27868]: audit 2023-10-09T04:07:08.443409+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:09 smithi153 bash[27868]: audit 2023-10-09T04:07:08.444766+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:09 smithi153 bash[27868]: audit 2023-10-09T04:07:08.446061+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:09 smithi153 bash[27868]: audit 2023-10-09T04:07:08.454711+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:09 smithi153 bash[27868]: cluster 2023-10-09T04:07:08.578216+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[27462]: audit 2023-10-09T04:07:08.443409+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[27462]: audit 2023-10-09T04:07:08.444766+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[27462]: audit 2023-10-09T04:07:08.446061+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[27462]: audit 2023-10-09T04:07:08.454711+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[27462]: cluster 2023-10-09T04:07:08.578216+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:09.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[33901]: audit 2023-10-09T04:07:08.443409+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[33901]: audit 2023-10-09T04:07:08.444766+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:09.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[33901]: audit 2023-10-09T04:07:08.446061+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:09.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[33901]: audit 2023-10-09T04:07:08.454711+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:09.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:09 smithi082 bash[33901]: cluster 2023-10-09T04:07:08.578216+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:10 smithi082 bash[27462]: audit 2023-10-09T04:07:09.865310+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:10 smithi082 bash[27462]: audit 2023-10-09T04:07:09.866105+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:10.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:10 smithi082 bash[33901]: audit 2023-10-09T04:07:09.865310+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:10 smithi082 bash[33901]: audit 2023-10-09T04:07:09.866105+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:10.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:10 smithi153 bash[27868]: audit 2023-10-09T04:07:09.865310+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:10.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:10 smithi153 bash[27868]: audit 2023-10-09T04:07:09.866105+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: audit 2023-10-09T04:07:10.468634+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: cluster 2023-10-09T04:07:10.468763+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: audit 2023-10-09T04:07:10.469009+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: audit 2023-10-09T04:07:10.469518+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: audit 2023-10-09T04:07:10.470172+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:11 smithi153 bash[27868]: cluster 2023-10-09T04:07:10.578745+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:11.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: audit 2023-10-09T04:07:10.468634+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:07:11.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: cluster 2023-10-09T04:07:10.468763+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T04:07:11.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: audit 2023-10-09T04:07:10.469009+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: audit 2023-10-09T04:07:10.469518+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: audit 2023-10-09T04:07:10.470172+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[27462]: cluster 2023-10-09T04:07:10.578745+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: audit 2023-10-09T04:07:10.468634+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: cluster 2023-10-09T04:07:10.468763+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: audit 2023-10-09T04:07:10.469009+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: audit 2023-10-09T04:07:10.469518+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: audit 2023-10-09T04:07:10.470172+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:11.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:11 smithi082 bash[33901]: cluster 2023-10-09T04:07:10.578745+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:07:11.739 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 5 on host 'smithi153' 2023-10-09T04:07:12.404 DEBUG:teuthology.orchestra.run.smithi153:osd.5> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T04:07:12.407 INFO:tasks.cephadm:Deploying osd.6 on smithi153 with /dev/vg_nvme/lv_2... 2023-10-09T04:07:12.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:07:12.635 INFO:journalctl@ceph.osd.5.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:07:12.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: audit 2023-10-09T04:07:11.470004+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:12.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: cluster 2023-10-09T04:07:11.470256+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:07:12.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: audit 2023-10-09T04:07:11.471067+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: audit 2023-10-09T04:07:11.476538+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: audit 2023-10-09T04:07:11.737223+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:12.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:12 smithi153 bash[27868]: audit 2023-10-09T04:07:11.737583+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: audit 2023-10-09T04:07:11.470004+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: cluster 2023-10-09T04:07:11.470256+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:07:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: audit 2023-10-09T04:07:11.471067+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: audit 2023-10-09T04:07:11.476538+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: audit 2023-10-09T04:07:11.737223+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[27462]: audit 2023-10-09T04:07:11.737583+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: audit 2023-10-09T04:07:11.470004+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: cluster 2023-10-09T04:07:11.470256+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: audit 2023-10-09T04:07:11.471067+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: audit 2023-10-09T04:07:11.476538+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: audit 2023-10-09T04:07:11.737223+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:12 smithi082 bash[33901]: audit 2023-10-09T04:07:11.737583+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:10.869532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:10.869606+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:12.477515+0000 mon.a (mon.0) 439 : cluster [INF] osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071] boot 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:12.477571+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: audit 2023-10-09T04:07:12.478447+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:13.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:12.579324+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:13.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:13 smithi153 bash[27868]: cluster 2023-10-09T04:07:13.474176+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T04:07:13.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:10.869532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:13.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:10.869606+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:13.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:12.477515+0000 mon.a (mon.0) 439 : cluster [INF] osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071] boot 2023-10-09T04:07:13.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:12.477571+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: audit 2023-10-09T04:07:12.478447+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:12.579324+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[27462]: cluster 2023-10-09T04:07:13.474176+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:10.869532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:10.869606+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:12.477515+0000 mon.a (mon.0) 439 : cluster [INF] osd.5 [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071] boot 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:12.477571+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: audit 2023-10-09T04:07:12.478447+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:07:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:12.579324+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:13.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:13 smithi082 bash[33901]: cluster 2023-10-09T04:07:13.474176+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T04:07:15.542 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:07:15.542 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:07:15.542 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2023-10-09T04:07:15.542 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2023-10-09T04:07:15.542 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.017878 s, 587 MB/s 2023-10-09T04:07:15.543 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2023-10-09T04:07:15.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:15 smithi153 bash[27868]: cluster 2023-10-09T04:07:14.476648+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:07:15.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:15 smithi153 bash[27868]: cluster 2023-10-09T04:07:14.579898+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:15.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:15 smithi082 bash[27462]: cluster 2023-10-09T04:07:14.476648+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:07:15.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:15 smithi082 bash[27462]: cluster 2023-10-09T04:07:14.579898+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:15.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:15 smithi082 bash[33901]: cluster 2023-10-09T04:07:14.476648+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:07:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:15 smithi082 bash[33901]: cluster 2023-10-09T04:07:14.579898+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:16.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi153:vg_nvme/lv_2 2023-10-09T04:07:17.859 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:17 smithi153 bash[27868]: cluster 2023-10-09T04:07:16.580414+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:17 smithi082 bash[27462]: cluster 2023-10-09T04:07:16.580414+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:17 smithi082 bash[33901]: cluster 2023-10-09T04:07:16.580414+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:19.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:19 smithi153 bash[27868]: cluster 2023-10-09T04:07:18.580956+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:19.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:19 smithi082 bash[27462]: cluster 2023-10-09T04:07:18.580956+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:19 smithi082 bash[33901]: cluster 2023-10-09T04:07:18.580956+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:20 smithi153 bash[27868]: audit 2023-10-09T04:07:20.471192+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:07:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:20 smithi153 bash[27868]: audit 2023-10-09T04:07:20.476649+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:07:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:20 smithi153 bash[27868]: audit 2023-10-09T04:07:20.478249+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:20.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[27462]: audit 2023-10-09T04:07:20.471192+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:07:20.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[27462]: audit 2023-10-09T04:07:20.476649+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:07:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[27462]: audit 2023-10-09T04:07:20.478249+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[33901]: audit 2023-10-09T04:07:20.471192+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:07:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[33901]: audit 2023-10-09T04:07:20.476649+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:07:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:20 smithi082 bash[33901]: audit 2023-10-09T04:07:20.478249+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:21.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:21 smithi153 bash[27868]: audit 2023-10-09T04:07:20.465121+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:21.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:21 smithi153 bash[27868]: cluster 2023-10-09T04:07:20.581550+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:21 smithi082 bash[27462]: audit 2023-10-09T04:07:20.465121+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:21 smithi082 bash[27462]: cluster 2023-10-09T04:07:20.581550+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:21 smithi082 bash[33901]: audit 2023-10-09T04:07:20.465121+0000 mgr.y (mgr.14146) 205 : audit [DBG] from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:21 smithi082 bash[33901]: cluster 2023-10-09T04:07:20.581550+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:23.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: cephadm 2023-10-09T04:07:21.809223+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.819183+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.820483+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.821671+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: cephadm 2023-10-09T04:07:21.822586+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8632M 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.829926+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.831524+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.832896+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:22 smithi153 bash[27868]: audit 2023-10-09T04:07:21.842668+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: cephadm 2023-10-09T04:07:21.809223+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:07:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.819183+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.820483+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.821671+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: cephadm 2023-10-09T04:07:21.822586+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8632M 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.829926+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.831524+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.832896+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[27462]: audit 2023-10-09T04:07:21.842668+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: cephadm 2023-10-09T04:07:21.809223+0000 mgr.y (mgr.14146) 207 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.819183+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.820483+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.821671+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: cephadm 2023-10-09T04:07:21.822586+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 8632M 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.829926+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.831524+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.832896+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:23.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:22 smithi082 bash[33901]: audit 2023-10-09T04:07:21.842668+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:24.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:23 smithi153 bash[27868]: cluster 2023-10-09T04:07:22.582128+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:24.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:23 smithi082 bash[27462]: cluster 2023-10-09T04:07:22.582128+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:24.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:23 smithi082 bash[33901]: cluster 2023-10-09T04:07:22.582128+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:25.252 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:24 smithi153 bash[27868]: cluster 2023-10-09T04:07:24.582687+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:25.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:24 smithi082 bash[27462]: cluster 2023-10-09T04:07:24.582687+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:25.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:24 smithi082 bash[33901]: cluster 2023-10-09T04:07:24.582687+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:27.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:27 smithi153 bash[27868]: cluster 2023-10-09T04:07:26.583232+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:27.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:27 smithi082 bash[27462]: cluster 2023-10-09T04:07:26.583232+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:27.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:27 smithi082 bash[33901]: cluster 2023-10-09T04:07:26.583232+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:28 smithi153 bash[27868]: audit 2023-10-09T04:07:28.453865+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.153:0/1213475526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:28 smithi153 bash[27868]: audit 2023-10-09T04:07:28.455189+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:28 smithi153 bash[27868]: audit 2023-10-09T04:07:28.463442+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]': finished 2023-10-09T04:07:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:28 smithi153 bash[27868]: cluster 2023-10-09T04:07:28.463557+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T04:07:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:28 smithi153 bash[27868]: audit 2023-10-09T04:07:28.463782+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[27462]: audit 2023-10-09T04:07:28.453865+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.153:0/1213475526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[27462]: audit 2023-10-09T04:07:28.455189+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[27462]: audit 2023-10-09T04:07:28.463442+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]': finished 2023-10-09T04:07:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[27462]: cluster 2023-10-09T04:07:28.463557+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T04:07:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[27462]: audit 2023-10-09T04:07:28.463782+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[33901]: audit 2023-10-09T04:07:28.453865+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.153:0/1213475526' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[33901]: audit 2023-10-09T04:07:28.455189+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]: dispatch 2023-10-09T04:07:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[33901]: audit 2023-10-09T04:07:28.463442+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b3727fa4-1f3c-4a8e-89b4-a9e631638f17"}]': finished 2023-10-09T04:07:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[33901]: cluster 2023-10-09T04:07:28.463557+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T04:07:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:28 smithi082 bash[33901]: audit 2023-10-09T04:07:28.463782+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:29 smithi153 bash[27868]: cluster 2023-10-09T04:07:28.583700+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:29 smithi153 bash[27868]: audit 2023-10-09T04:07:29.185811+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.153:0/1543758077' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:07:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:29 smithi082 bash[27462]: cluster 2023-10-09T04:07:28.583700+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:29 smithi082 bash[27462]: audit 2023-10-09T04:07:29.185811+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.153:0/1543758077' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:07:29.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:29 smithi082 bash[33901]: cluster 2023-10-09T04:07:28.583700+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:29.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:29 smithi082 bash[33901]: audit 2023-10-09T04:07:29.185811+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.153:0/1543758077' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:07:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:31 smithi153 bash[27868]: cluster 2023-10-09T04:07:30.584267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:31 smithi082 bash[27462]: cluster 2023-10-09T04:07:30.584267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:31.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:31 smithi082 bash[33901]: cluster 2023-10-09T04:07:30.584267+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:33.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:33 smithi153 bash[27868]: cluster 2023-10-09T04:07:32.584857+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:33 smithi082 bash[27462]: cluster 2023-10-09T04:07:32.584857+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:33.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:33 smithi082 bash[33901]: cluster 2023-10-09T04:07:32.584857+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:35.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:34 smithi153 bash[27868]: cluster 2023-10-09T04:07:34.585404+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:35.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:34 smithi082 bash[33901]: cluster 2023-10-09T04:07:34.585404+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:35.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:34 smithi082 bash[27462]: cluster 2023-10-09T04:07:34.585404+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:37.910 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:37 smithi153 bash[27868]: cluster 2023-10-09T04:07:36.585925+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:37 smithi082 bash[27462]: cluster 2023-10-09T04:07:36.585925+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:37 smithi082 bash[33901]: cluster 2023-10-09T04:07:36.585925+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:39 smithi153 bash[27868]: cluster 2023-10-09T04:07:38.586501+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:39 smithi082 bash[27462]: cluster 2023-10-09T04:07:38.586501+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:39.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:39 smithi082 bash[33901]: cluster 2023-10-09T04:07:38.586501+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:40.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:40 smithi153 bash[27868]: audit 2023-10-09T04:07:40.616916+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:07:40.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:40 smithi153 bash[27868]: audit 2023-10-09T04:07:40.618561+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:40.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:40 smithi082 bash[27462]: audit 2023-10-09T04:07:40.616916+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:07:40.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:40 smithi082 bash[27462]: audit 2023-10-09T04:07:40.618561+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:40 smithi082 bash[33901]: audit 2023-10-09T04:07:40.616916+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:07:40.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:40 smithi082 bash[33901]: audit 2023-10-09T04:07:40.618561+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:41.917 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:41 smithi153 bash[27868]: cluster 2023-10-09T04:07:40.587030+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:41.917 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:41 smithi153 bash[27868]: cephadm 2023-10-09T04:07:40.619989+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi153 2023-10-09T04:07:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:41 smithi082 bash[33901]: cluster 2023-10-09T04:07:40.587030+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:41 smithi082 bash[33901]: cephadm 2023-10-09T04:07:40.619989+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi153 2023-10-09T04:07:41.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:41 smithi082 bash[27462]: cluster 2023-10-09T04:07:40.587030+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:41.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:41 smithi082 bash[27462]: cephadm 2023-10-09T04:07:40.619989+0000 mgr.y (mgr.14146) 219 : cephadm [INF] Deploying daemon osd.6 on smithi153 2023-10-09T04:07:43.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:43 smithi153 bash[27868]: cluster 2023-10-09T04:07:42.587582+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:43 smithi082 bash[27462]: cluster 2023-10-09T04:07:42.587582+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:43.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:43 smithi082 bash[33901]: cluster 2023-10-09T04:07:42.587582+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:45 smithi153 bash[27868]: cluster 2023-10-09T04:07:44.588130+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:45 smithi153 bash[27868]: audit 2023-10-09T04:07:44.653753+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:45 smithi153 bash[27868]: audit 2023-10-09T04:07:44.671048+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[27462]: cluster 2023-10-09T04:07:44.588130+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[27462]: audit 2023-10-09T04:07:44.653753+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:45.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[27462]: audit 2023-10-09T04:07:44.671048+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:45.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[33901]: cluster 2023-10-09T04:07:44.588130+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:45.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[33901]: audit 2023-10-09T04:07:44.653753+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:45.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:45 smithi082 bash[33901]: audit 2023-10-09T04:07:44.671048+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:47 smithi153 bash[27868]: cluster 2023-10-09T04:07:46.588665+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:47.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:47 smithi082 bash[27462]: cluster 2023-10-09T04:07:46.588665+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:47.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:47 smithi082 bash[33901]: cluster 2023-10-09T04:07:46.588665+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:49 smithi153 bash[27868]: cluster 2023-10-09T04:07:48.589178+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:49.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:49 smithi082 bash[33901]: cluster 2023-10-09T04:07:48.589178+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:49 smithi082 bash[27462]: cluster 2023-10-09T04:07:48.589178+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:51.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:50 smithi153 bash[27868]: audit 2023-10-09T04:07:49.814735+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:51.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:50 smithi153 bash[27868]: audit 2023-10-09T04:07:50.089600+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:51.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:50 smithi153 bash[27868]: audit 2023-10-09T04:07:50.091292+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:51.172 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:50 smithi153 bash[27868]: audit 2023-10-09T04:07:50.099414+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[27462]: audit 2023-10-09T04:07:49.814735+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[27462]: audit 2023-10-09T04:07:50.089600+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[27462]: audit 2023-10-09T04:07:50.091292+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:51.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[27462]: audit 2023-10-09T04:07:50.099414+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[33901]: audit 2023-10-09T04:07:49.814735+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[33901]: audit 2023-10-09T04:07:50.089600+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[33901]: audit 2023-10-09T04:07:50.091292+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:50 smithi082 bash[33901]: audit 2023-10-09T04:07:50.099414+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:52.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:51 smithi153 bash[27868]: cluster 2023-10-09T04:07:50.589767+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:52.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:51 smithi153 bash[27868]: audit 2023-10-09T04:07:51.260287+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:52.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:51 smithi153 bash[27868]: audit 2023-10-09T04:07:51.261226+0000 mon.a (mon.0) 466 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[33901]: cluster 2023-10-09T04:07:50.589767+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[33901]: audit 2023-10-09T04:07:51.260287+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[33901]: audit 2023-10-09T04:07:51.261226+0000 mon.a (mon.0) 466 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:52.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[27462]: cluster 2023-10-09T04:07:50.589767+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:52.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[27462]: audit 2023-10-09T04:07:51.260287+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:52.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:51 smithi082 bash[27462]: audit 2023-10-09T04:07:51.261226+0000 mon.a (mon.0) 466 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:07:53.375 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 6 on host 'smithi153' 2023-10-09T04:07:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: audit 2023-10-09T04:07:52.105304+0000 mon.a (mon.0) 467 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:07:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: cluster 2023-10-09T04:07:52.105432+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T04:07:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: audit 2023-10-09T04:07:52.105670+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: audit 2023-10-09T04:07:52.106508+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: audit 2023-10-09T04:07:52.107284+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:53 smithi153 bash[27868]: cluster 2023-10-09T04:07:52.590372+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:53.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: audit 2023-10-09T04:07:52.105304+0000 mon.a (mon.0) 467 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:07:53.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: cluster 2023-10-09T04:07:52.105432+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T04:07:53.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: audit 2023-10-09T04:07:52.105670+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: audit 2023-10-09T04:07:52.106508+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: audit 2023-10-09T04:07:52.107284+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[27462]: cluster 2023-10-09T04:07:52.590372+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: audit 2023-10-09T04:07:52.105304+0000 mon.a (mon.0) 467 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: cluster 2023-10-09T04:07:52.105432+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: audit 2023-10-09T04:07:52.105670+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: audit 2023-10-09T04:07:52.106508+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: audit 2023-10-09T04:07:52.107284+0000 mon.a (mon.0) 470 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:07:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:53 smithi082 bash[33901]: cluster 2023-10-09T04:07:52.590372+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:07:54.089 DEBUG:teuthology.orchestra.run.smithi153:osd.6> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T04:07:54.091 INFO:tasks.cephadm:Deploying osd.7 on smithi153 with /dev/vg_nvme/lv_1... 2023-10-09T04:07:54.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:07:54.185 INFO:journalctl@ceph.osd.6.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:07:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.112672+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: cluster 2023-10-09T04:07:53.112936+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T04:07:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.113589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.119898+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.373478+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.376404+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: cluster 2023-10-09T04:07:53.993157+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505] boot 2023-10-09T04:07:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: cluster 2023-10-09T04:07:53.993259+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T04:07:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:54 smithi153 bash[27868]: audit 2023-10-09T04:07:53.994362+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.112672+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: cluster 2023-10-09T04:07:53.112936+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.113589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.119898+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.373478+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.376404+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: cluster 2023-10-09T04:07:53.993157+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505] boot 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: cluster 2023-10-09T04:07:53.993259+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[27462]: audit 2023-10-09T04:07:53.994362+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.112672+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: cluster 2023-10-09T04:07:53.112936+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.113589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.119898+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.373478+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.376404+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: cluster 2023-10-09T04:07:53.993157+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505] boot 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: cluster 2023-10-09T04:07:53.993259+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T04:07:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:54 smithi082 bash[33901]: audit 2023-10-09T04:07:53.994362+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:07:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:55 smithi153 bash[27868]: cluster 2023-10-09T04:07:52.273278+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:55 smithi153 bash[27868]: cluster 2023-10-09T04:07:52.273385+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:55 smithi153 bash[27868]: cluster 2023-10-09T04:07:54.591018+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:55.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:55 smithi153 bash[27868]: cluster 2023-10-09T04:07:54.995867+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T04:07:55.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[27462]: cluster 2023-10-09T04:07:52.273278+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:55.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[27462]: cluster 2023-10-09T04:07:52.273385+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:55.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[27462]: cluster 2023-10-09T04:07:54.591018+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:55.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[27462]: cluster 2023-10-09T04:07:54.995867+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T04:07:55.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[33901]: cluster 2023-10-09T04:07:52.273278+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:07:55.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[33901]: cluster 2023-10-09T04:07:52.273385+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:07:55.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[33901]: cluster 2023-10-09T04:07:54.591018+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:55.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:55 smithi082 bash[33901]: cluster 2023-10-09T04:07:54.995867+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0176346 s, 595 MB/s 2023-10-09T04:07:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2023-10-09T04:07:57.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:57 smithi153 bash[27868]: cluster 2023-10-09T04:07:55.998289+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T04:07:57.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:57 smithi153 bash[27868]: cluster 2023-10-09T04:07:56.591679+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:57 smithi082 bash[27462]: cluster 2023-10-09T04:07:55.998289+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T04:07:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:57 smithi082 bash[27462]: cluster 2023-10-09T04:07:56.591679+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:57.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:57 smithi082 bash[33901]: cluster 2023-10-09T04:07:55.998289+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T04:07:57.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:57 smithi082 bash[33901]: cluster 2023-10-09T04:07:56.591679+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v215: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:58.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi153:vg_nvme/lv_1 2023-10-09T04:07:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:58 smithi153 bash[27868]: cluster 2023-10-09T04:07:56.992843+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:07:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:58 smithi082 bash[27462]: cluster 2023-10-09T04:07:56.992843+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:07:58.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:58 smithi082 bash[33901]: cluster 2023-10-09T04:07:56.992843+0000 mon.a (mon.0) 482 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:07:59.407 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:07:59 smithi153 bash[27868]: cluster 2023-10-09T04:07:58.592339+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:59.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:07:59 smithi082 bash[27462]: cluster 2023-10-09T04:07:58.592339+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:07:59.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:07:59 smithi082 bash[33901]: cluster 2023-10-09T04:07:58.592339+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v216: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:00 smithi082 bash[27462]: cluster 2023-10-09T04:08:00.644142+0000 mon.a (mon.0) 483 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:08:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:00 smithi082 bash[27462]: cluster 2023-10-09T04:08:00.644198+0000 mon.a (mon.0) 484 : cluster [INF] Cluster is now healthy 2023-10-09T04:08:00.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:00 smithi082 bash[33901]: cluster 2023-10-09T04:08:00.644142+0000 mon.a (mon.0) 483 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:08:00.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:00 smithi082 bash[33901]: cluster 2023-10-09T04:08:00.644198+0000 mon.a (mon.0) 484 : cluster [INF] Cluster is now healthy 2023-10-09T04:08:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:00 smithi153 bash[27868]: cluster 2023-10-09T04:08:00.644142+0000 mon.a (mon.0) 483 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:08:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:00 smithi153 bash[27868]: cluster 2023-10-09T04:08:00.644198+0000 mon.a (mon.0) 484 : cluster [INF] Cluster is now healthy 2023-10-09T04:08:01.959 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:01 smithi153 bash[27868]: cluster 2023-10-09T04:08:00.592861+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:01.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:01 smithi082 bash[27462]: cluster 2023-10-09T04:08:00.592861+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:01 smithi082 bash[33901]: cluster 2023-10-09T04:08:00.592861+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[27462]: audit 2023-10-09T04:08:02.366649+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:08:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[27462]: audit 2023-10-09T04:08:02.371643+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:08:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[27462]: audit 2023-10-09T04:08:02.373131+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[33901]: audit 2023-10-09T04:08:02.366649+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:08:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[33901]: audit 2023-10-09T04:08:02.371643+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:08:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:02 smithi082 bash[33901]: audit 2023-10-09T04:08:02.373131+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:02.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:02 smithi153 bash[27868]: audit 2023-10-09T04:08:02.366649+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:08:02.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:02 smithi153 bash[27868]: audit 2023-10-09T04:08:02.371643+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:08:02.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:02 smithi153 bash[27868]: audit 2023-10-09T04:08:02.373131+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:03 smithi082 bash[27462]: audit 2023-10-09T04:08:02.360237+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:08:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:03 smithi082 bash[27462]: cluster 2023-10-09T04:08:02.593428+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:03 smithi082 bash[33901]: audit 2023-10-09T04:08:02.360237+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:08:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:03 smithi082 bash[33901]: cluster 2023-10-09T04:08:02.593428+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:04.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:03 smithi153 bash[27868]: audit 2023-10-09T04:08:02.360237+0000 mgr.y (mgr.14146) 230 : audit [DBG] from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:08:04.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:03 smithi153 bash[27868]: cluster 2023-10-09T04:08:02.593428+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:04.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: cephadm 2023-10-09T04:08:03.718932+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.728494+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.729966+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.731361+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.732852+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: cephadm 2023-10-09T04:08:03.733997+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5755M 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.743856+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.745454+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.746761+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[27462]: audit 2023-10-09T04:08:03.756313+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: cephadm 2023-10-09T04:08:03.718932+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.728494+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.729966+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.731361+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.732852+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: cephadm 2023-10-09T04:08:03.733997+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5755M 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.743856+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.745454+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.746761+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:04 smithi082 bash[33901]: audit 2023-10-09T04:08:03.756313+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: cephadm 2023-10-09T04:08:03.718932+0000 mgr.y (mgr.14146) 232 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.728494+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.729966+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.731361+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.732852+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: cephadm 2023-10-09T04:08:03.733997+0000 mgr.y (mgr.14146) 233 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 5755M 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.743856+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.745454+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.746761+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:04 smithi153 bash[27868]: audit 2023-10-09T04:08:03.756313+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:06.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:05 smithi153 bash[27868]: cluster 2023-10-09T04:08:04.594010+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:05 smithi082 bash[27462]: cluster 2023-10-09T04:08:04.594010+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:06.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:05 smithi082 bash[33901]: cluster 2023-10-09T04:08:04.594010+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:08.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:07 smithi153 bash[27868]: cluster 2023-10-09T04:08:06.594560+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:08.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:07 smithi082 bash[27462]: cluster 2023-10-09T04:08:06.594560+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:08.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:07 smithi082 bash[33901]: cluster 2023-10-09T04:08:06.594560+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:10.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:09 smithi153 bash[27868]: cluster 2023-10-09T04:08:08.595099+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:09 smithi082 bash[27462]: cluster 2023-10-09T04:08:08.595099+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:10.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:09 smithi082 bash[33901]: cluster 2023-10-09T04:08:08.595099+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:11.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:10 smithi153 bash[27868]: audit 2023-10-09T04:08:10.301136+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.153:0/3752434756' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:10 smithi153 bash[27868]: audit 2023-10-09T04:08:10.302261+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:10 smithi153 bash[27868]: audit 2023-10-09T04:08:10.310090+0000 mon.a (mon.0) 497 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]': finished 2023-10-09T04:08:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:10 smithi153 bash[27868]: cluster 2023-10-09T04:08:10.310167+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T04:08:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:10 smithi153 bash[27868]: audit 2023-10-09T04:08:10.310333+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:11.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[27462]: audit 2023-10-09T04:08:10.301136+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.153:0/3752434756' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[27462]: audit 2023-10-09T04:08:10.302261+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[27462]: audit 2023-10-09T04:08:10.310090+0000 mon.a (mon.0) 497 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]': finished 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[27462]: cluster 2023-10-09T04:08:10.310167+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[27462]: audit 2023-10-09T04:08:10.310333+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[33901]: audit 2023-10-09T04:08:10.301136+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.153:0/3752434756' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[33901]: audit 2023-10-09T04:08:10.302261+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]: dispatch 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[33901]: audit 2023-10-09T04:08:10.310090+0000 mon.a (mon.0) 497 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ae5ef44-7804-4aa6-9343-0cd93c170de2"}]': finished 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[33901]: cluster 2023-10-09T04:08:10.310167+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T04:08:11.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:10 smithi082 bash[33901]: audit 2023-10-09T04:08:10.310333+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:11 smithi153 bash[27868]: cluster 2023-10-09T04:08:10.595635+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:11 smithi153 bash[27868]: audit 2023-10-09T04:08:11.016744+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.153:0/3420783333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:08:12.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:11 smithi082 bash[27462]: cluster 2023-10-09T04:08:10.595635+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:12.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:11 smithi082 bash[27462]: audit 2023-10-09T04:08:11.016744+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.153:0/3420783333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:08:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:11 smithi082 bash[33901]: cluster 2023-10-09T04:08:10.595635+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:11 smithi082 bash[33901]: audit 2023-10-09T04:08:11.016744+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.153:0/3420783333' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:08:13.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:13 smithi153 bash[27868]: cluster 2023-10-09T04:08:12.596201+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:14.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:13 smithi082 bash[27462]: cluster 2023-10-09T04:08:12.596201+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:14.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:13 smithi082 bash[33901]: cluster 2023-10-09T04:08:12.596201+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:15 smithi153 bash[27868]: cluster 2023-10-09T04:08:14.596828+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:15 smithi082 bash[27462]: cluster 2023-10-09T04:08:14.596828+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:15.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:15 smithi082 bash[33901]: cluster 2023-10-09T04:08:14.596828+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:17 smithi153 bash[27868]: cluster 2023-10-09T04:08:16.597379+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:17 smithi082 bash[27462]: cluster 2023-10-09T04:08:16.597379+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:17.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:17 smithi082 bash[33901]: cluster 2023-10-09T04:08:16.597379+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:19.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:19 smithi153 bash[27868]: cluster 2023-10-09T04:08:18.597942+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:19.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:19 smithi082 bash[27462]: cluster 2023-10-09T04:08:18.597942+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:19.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:19 smithi082 bash[33901]: cluster 2023-10-09T04:08:18.597942+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:21.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:21 smithi153 bash[27868]: cluster 2023-10-09T04:08:20.598489+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:21 smithi082 bash[27462]: cluster 2023-10-09T04:08:20.598489+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:21 smithi082 bash[33901]: cluster 2023-10-09T04:08:20.598489+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:22 smithi153 bash[27868]: audit 2023-10-09T04:08:22.488680+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:08:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:22 smithi153 bash[27868]: audit 2023-10-09T04:08:22.490602+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:22 smithi082 bash[27462]: audit 2023-10-09T04:08:22.488680+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:08:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:22 smithi082 bash[27462]: audit 2023-10-09T04:08:22.490602+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:22 smithi082 bash[33901]: audit 2023-10-09T04:08:22.488680+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:08:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:22 smithi082 bash[33901]: audit 2023-10-09T04:08:22.490602+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:23.835 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:23 smithi153 bash[27868]: cephadm 2023-10-09T04:08:22.492098+0000 mgr.y (mgr.14146) 243 : cephadm [INF] Deploying daemon osd.7 on smithi153 2023-10-09T04:08:23.835 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:23 smithi153 bash[27868]: cluster 2023-10-09T04:08:22.599020+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:23.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:23 smithi082 bash[27462]: cephadm 2023-10-09T04:08:22.492098+0000 mgr.y (mgr.14146) 243 : cephadm [INF] Deploying daemon osd.7 on smithi153 2023-10-09T04:08:24.022 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:23 smithi082 bash[27462]: cluster 2023-10-09T04:08:22.599020+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:24.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:23 smithi082 bash[33901]: cephadm 2023-10-09T04:08:22.492098+0000 mgr.y (mgr.14146) 243 : cephadm [INF] Deploying daemon osd.7 on smithi153 2023-10-09T04:08:24.024 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:23 smithi082 bash[33901]: cluster 2023-10-09T04:08:22.599020+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:25.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:25 smithi153 bash[27868]: cluster 2023-10-09T04:08:24.599579+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:25.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:25 smithi082 bash[27462]: cluster 2023-10-09T04:08:24.599579+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:25.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:25 smithi082 bash[33901]: cluster 2023-10-09T04:08:24.599579+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:27 smithi153 bash[27868]: audit 2023-10-09T04:08:26.568174+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:27 smithi153 bash[27868]: audit 2023-10-09T04:08:26.588939+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:27 smithi153 bash[27868]: cluster 2023-10-09T04:08:26.600089+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:27.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[27462]: audit 2023-10-09T04:08:26.568174+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:27.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[27462]: audit 2023-10-09T04:08:26.588939+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[27462]: cluster 2023-10-09T04:08:26.600089+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[33901]: audit 2023-10-09T04:08:26.568174+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[33901]: audit 2023-10-09T04:08:26.588939+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:27 smithi082 bash[33901]: cluster 2023-10-09T04:08:26.600089+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:29 smithi153 bash[27868]: cluster 2023-10-09T04:08:28.600629+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:29 smithi082 bash[27462]: cluster 2023-10-09T04:08:28.600629+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:29.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:29 smithi082 bash[33901]: cluster 2023-10-09T04:08:28.600629+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:31 smithi153 bash[27868]: cluster 2023-10-09T04:08:30.601238+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:31 smithi082 bash[27462]: cluster 2023-10-09T04:08:30.601238+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:31.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:31 smithi082 bash[33901]: cluster 2023-10-09T04:08:30.601238+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:33.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:32 smithi153 bash[27868]: audit 2023-10-09T04:08:31.785109+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:33.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:32 smithi153 bash[27868]: audit 2023-10-09T04:08:31.786445+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:33.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:32 smithi153 bash[27868]: audit 2023-10-09T04:08:31.787930+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:33.052 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:32 smithi153 bash[27868]: audit 2023-10-09T04:08:31.796446+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:33.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[27462]: audit 2023-10-09T04:08:31.785109+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:33.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[27462]: audit 2023-10-09T04:08:31.786445+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[27462]: audit 2023-10-09T04:08:31.787930+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[27462]: audit 2023-10-09T04:08:31.796446+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[33901]: audit 2023-10-09T04:08:31.785109+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[33901]: audit 2023-10-09T04:08:31.786445+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[33901]: audit 2023-10-09T04:08:31.787930+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:33.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:32 smithi082 bash[33901]: audit 2023-10-09T04:08:31.796446+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:33 smithi153 bash[27868]: cluster 2023-10-09T04:08:32.601753+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:33 smithi153 bash[27868]: audit 2023-10-09T04:08:33.054962+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:33 smithi153 bash[27868]: audit 2023-10-09T04:08:33.055765+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[33901]: cluster 2023-10-09T04:08:32.601753+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:34.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[33901]: audit 2023-10-09T04:08:33.054962+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[33901]: audit 2023-10-09T04:08:33.055765+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[27462]: cluster 2023-10-09T04:08:32.601753+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[27462]: audit 2023-10-09T04:08:33.054962+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:33 smithi082 bash[27462]: audit 2023-10-09T04:08:33.055765+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:08:34.995 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 7 on host 'smithi153' 2023-10-09T04:08:35.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:34 smithi153 bash[27868]: audit 2023-10-09T04:08:33.810106+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:08:35.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:34 smithi153 bash[27868]: cluster 2023-10-09T04:08:33.810212+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T04:08:35.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:34 smithi153 bash[27868]: audit 2023-10-09T04:08:33.810422+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:34 smithi153 bash[27868]: audit 2023-10-09T04:08:33.810989+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:34 smithi153 bash[27868]: audit 2023-10-09T04:08:33.811526+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[33901]: audit 2023-10-09T04:08:33.810106+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:08:35.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[33901]: cluster 2023-10-09T04:08:33.810212+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[33901]: audit 2023-10-09T04:08:33.810422+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[33901]: audit 2023-10-09T04:08:33.810989+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[33901]: audit 2023-10-09T04:08:33.811526+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[27462]: audit 2023-10-09T04:08:33.810106+0000 mon.a (mon.0) 509 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[27462]: cluster 2023-10-09T04:08:33.810212+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[27462]: audit 2023-10-09T04:08:33.810422+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[27462]: audit 2023-10-09T04:08:33.810989+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:34 smithi082 bash[27462]: audit 2023-10-09T04:08:33.811526+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:08:35.644 DEBUG:teuthology.orchestra.run.smithi153:osd.7> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T04:08:35.646 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T04:08:35.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T04:08:35.818 INFO:journalctl@ceph.osd.7.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:08:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: cluster 2023-10-09T04:08:34.602316+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: audit 2023-10-09T04:08:34.812429+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: cluster 2023-10-09T04:08:34.812564+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: audit 2023-10-09T04:08:34.813142+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: audit 2023-10-09T04:08:34.819175+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: audit 2023-10-09T04:08:34.993527+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:35 smithi153 bash[27868]: audit 2023-10-09T04:08:35.023744+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:35.996 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: cluster 2023-10-09T04:08:34.602316+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:35.996 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: audit 2023-10-09T04:08:34.812429+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: cluster 2023-10-09T04:08:34.812564+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: audit 2023-10-09T04:08:34.813142+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: audit 2023-10-09T04:08:34.819175+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: audit 2023-10-09T04:08:34.993527+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[27462]: audit 2023-10-09T04:08:35.023744+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: cluster 2023-10-09T04:08:34.602316+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: audit 2023-10-09T04:08:34.812429+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: cluster 2023-10-09T04:08:34.812564+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T04:08:35.997 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: audit 2023-10-09T04:08:34.813142+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: audit 2023-10-09T04:08:34.819175+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:35.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: audit 2023-10-09T04:08:34.993527+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:35.998 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:35 smithi082 bash[33901]: audit 2023-10-09T04:08:35.023744+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:37.110 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: cluster 2023-10-09T04:08:34.086081+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:08:37.110 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: cluster 2023-10-09T04:08:34.086164+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:08:37.110 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: cluster 2023-10-09T04:08:35.823097+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403] boot 2023-10-09T04:08:37.110 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: cluster 2023-10-09T04:08:35.823281+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: audit 2023-10-09T04:08:35.824429+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[27462]: cluster 2023-10-09T04:08:36.824303+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: cluster 2023-10-09T04:08:34.086081+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: cluster 2023-10-09T04:08:34.086164+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: cluster 2023-10-09T04:08:35.823097+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403] boot 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: cluster 2023-10-09T04:08:35.823281+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: audit 2023-10-09T04:08:35.824429+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:37.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:36 smithi082 bash[33901]: cluster 2023-10-09T04:08:36.824303+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T04:08:37.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: cluster 2023-10-09T04:08:34.086081+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T04:08:37.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: cluster 2023-10-09T04:08:34.086164+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T04:08:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: cluster 2023-10-09T04:08:35.823097+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403] boot 2023-10-09T04:08:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: cluster 2023-10-09T04:08:35.823281+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T04:08:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: audit 2023-10-09T04:08:35.824429+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:08:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:36 smithi153 bash[27868]: cluster 2023-10-09T04:08:36.824303+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T04:08:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:37 smithi082 bash[33901]: cluster 2023-10-09T04:08:36.602920+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:37 smithi082 bash[33901]: cluster 2023-10-09T04:08:37.825551+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T04:08:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:37 smithi082 bash[27462]: cluster 2023-10-09T04:08:36.602920+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:37 smithi082 bash[27462]: cluster 2023-10-09T04:08:37.825551+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T04:08:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:37 smithi153 bash[27868]: cluster 2023-10-09T04:08:36.602920+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:08:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:37 smithi153 bash[27868]: cluster 2023-10-09T04:08:37.825551+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T04:08:38.711 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:08:39.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:38 smithi153 bash[27868]: audit 2023-10-09T04:08:38.709688+0000 mon.a (mon.0) 524 : audit [DBG] from='client.? 172.21.15.82:0/4070584940' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:08:39.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:38 smithi082 bash[27462]: audit 2023-10-09T04:08:38.709688+0000 mon.a (mon.0) 524 : audit [DBG] from='client.? 172.21.15.82:0/4070584940' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:08:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:38 smithi082 bash[33901]: audit 2023-10-09T04:08:38.709688+0000 mon.a (mon.0) 524 : audit [DBG] from='client.? 172.21.15.82:0/4070584940' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:08:39.423 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824515,"num_in_osds":8,"osd_in_since":1696824490,"num_remapped_pgs":0} 2023-10-09T04:08:39.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:08:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:39 smithi153 bash[27868]: cluster 2023-10-09T04:08:38.603482+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:39 smithi082 bash[27462]: cluster 2023-10-09T04:08:38.603482+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:39 smithi082 bash[33901]: cluster 2023-10-09T04:08:38.603482+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:41 smithi153 bash[27868]: cluster 2023-10-09T04:08:40.604041+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:41 smithi082 bash[27462]: cluster 2023-10-09T04:08:40.604041+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:41 smithi082 bash[33901]: cluster 2023-10-09T04:08:40.604041+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:44.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: cluster 2023-10-09T04:08:42.604642+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.773425+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.775149+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.776347+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.777351+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.778268+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.783877+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.785472+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.786755+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:43 smithi153 bash[27868]: audit 2023-10-09T04:08:43.792926+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.178 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: cluster 2023-10-09T04:08:42.604642+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.773425+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.775149+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.776347+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.777351+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.778268+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.783877+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.785472+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.786755+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[27462]: audit 2023-10-09T04:08:43.792926+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: cluster 2023-10-09T04:08:42.604642+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.773425+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.775149+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.776347+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.777351+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.778268+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:08:44.209 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.783877+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:44.209 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.785472+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:44.209 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.786755+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:44.209 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:43 smithi082 bash[33901]: audit 2023-10-09T04:08:43.792926+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:08:45.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:44 smithi082 bash[27462]: cephadm 2023-10-09T04:08:43.763285+0000 mgr.y (mgr.14146) 255 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:45.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:44 smithi082 bash[27462]: cephadm 2023-10-09T04:08:43.778986+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4316M 2023-10-09T04:08:45.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:44 smithi082 bash[33901]: cephadm 2023-10-09T04:08:43.763285+0000 mgr.y (mgr.14146) 255 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:45.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:44 smithi082 bash[33901]: cephadm 2023-10-09T04:08:43.778986+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4316M 2023-10-09T04:08:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:44 smithi153 bash[27868]: cephadm 2023-10-09T04:08:43.763285+0000 mgr.y (mgr.14146) 255 : cephadm [INF] Detected new or changed devices on smithi153 2023-10-09T04:08:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:44 smithi153 bash[27868]: cephadm 2023-10-09T04:08:43.778986+0000 mgr.y (mgr.14146) 256 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 4316M 2023-10-09T04:08:46.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:45 smithi153 bash[27868]: cluster 2023-10-09T04:08:44.605266+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:46.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:45 smithi082 bash[27462]: cluster 2023-10-09T04:08:44.605266+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:46.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:45 smithi082 bash[33901]: cluster 2023-10-09T04:08:44.605266+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:46.983 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:08:46.984 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":50,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:08:37.818560+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:08:47.644 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T04:05:08.936893+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T04:08:47.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T04:08:47.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:47 smithi082 bash[27462]: cluster 2023-10-09T04:08:46.605849+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.173 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:47 smithi082 bash[27462]: audit 2023-10-09T04:08:46.980919+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.82:0/2306074566' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:08:48.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:47 smithi082 bash[33901]: cluster 2023-10-09T04:08:46.605849+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:47 smithi082 bash[33901]: audit 2023-10-09T04:08:46.980919+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.82:0/2306074566' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:08:48.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:47 smithi153 bash[27868]: cluster 2023-10-09T04:08:46.605849+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:47 smithi153 bash[27868]: audit 2023-10-09T04:08:46.980919+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.82:0/2306074566' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:08:50.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:49 smithi153 bash[27868]: cluster 2023-10-09T04:08:48.606444+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:49 smithi082 bash[27462]: cluster 2023-10-09T04:08:48.606444+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:50.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:49 smithi082 bash[33901]: cluster 2023-10-09T04:08:48.606444+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:51 smithi153 bash[27868]: cluster 2023-10-09T04:08:50.607035+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:51 smithi082 bash[27462]: cluster 2023-10-09T04:08:50.607035+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:51 smithi082 bash[33901]: cluster 2023-10-09T04:08:50.607035+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.404 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:54.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:53 smithi153 bash[27868]: cluster 2023-10-09T04:08:52.607642+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:53 smithi082 bash[27462]: cluster 2023-10-09T04:08:52.607642+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:53 smithi082 bash[33901]: cluster 2023-10-09T04:08:52.607642+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:55.192 INFO:teuthology.orchestra.run.smithi082.stdout:pg_num: 1 2023-10-09T04:08:55.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:55 smithi082 bash[33901]: cluster 2023-10-09T04:08:54.608186+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:55 smithi082 bash[27462]: cluster 2023-10-09T04:08:54.608186+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:55 smithi153 bash[27868]: cluster 2023-10-09T04:08:54.608186+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:55.898 INFO:tasks.cephadm:Adding prometheus.a on smithi153 2023-10-09T04:08:55.898 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi153=a' 2023-10-09T04:08:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:56 smithi153 bash[27868]: audit 2023-10-09T04:08:55.191471+0000 mon.a (mon.0) 534 : audit [DBG] from='client.? 172.21.15.82:0/3552183691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T04:08:56.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:56 smithi082 bash[27462]: audit 2023-10-09T04:08:55.191471+0000 mon.a (mon.0) 534 : audit [DBG] from='client.? 172.21.15.82:0/3552183691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T04:08:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:56 smithi082 bash[33901]: audit 2023-10-09T04:08:55.191471+0000 mon.a (mon.0) 534 : audit [DBG] from='client.? 172.21.15.82:0/3552183691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T04:08:57.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:57 smithi153 bash[27868]: cluster 2023-10-09T04:08:56.608823+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:57 smithi082 bash[27462]: cluster 2023-10-09T04:08:56.608823+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:57.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:57 smithi082 bash[33901]: cluster 2023-10-09T04:08:56.608823+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:58.806 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled prometheus update... 2023-10-09T04:08:59.487 DEBUG:teuthology.orchestra.run.smithi153:prometheus.a> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T04:08:59.489 INFO:tasks.cephadm:Adding node-exporter.a on smithi082 2023-10-09T04:08:59.489 INFO:tasks.cephadm:Adding node-exporter.b on smithi153 2023-10-09T04:08:59.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi082=a;smithi153=b' 2023-10-09T04:08:59.682 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:09:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: cluster 2023-10-09T04:08:58.609468+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:58.793936+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: cephadm 2023-10-09T04:08:58.797644+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:58.804326+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:58.850007+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:59.148460+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:59.150225+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:59.160732+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:08:59 smithi153 bash[27868]: audit 2023-10-09T04:08:59.168950+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:09:00.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: cluster 2023-10-09T04:08:58.609468+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:58.793936+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:00.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: cephadm 2023-10-09T04:08:58.797644+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2023-10-09T04:09:00.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:58.804326+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:58.850007+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:59.148460+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:59.150225+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:59.160732+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[27462]: audit 2023-10-09T04:08:59.168950+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: cluster 2023-10-09T04:08:58.609468+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:58.793936+0000 mgr.y (mgr.14146) 265 : audit [DBG] from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: cephadm 2023-10-09T04:08:58.797644+0000 mgr.y (mgr.14146) 266 : cephadm [INF] Saving service prometheus spec with placement smithi153=a;count:1 2023-10-09T04:09:00.212 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:58.804326+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.213 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:58.850007+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:00.213 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:59.148460+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.213 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:59.150225+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.213 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:59.160732+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' 2023-10-09T04:09:00.213 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:08:59 smithi082 bash[33901]: audit 2023-10-09T04:08:59.168950+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:09:00.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: ignoring --setuser ceph since I am not root 2023-10-09T04:09:00.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: ignoring --setgroup ceph since I am not root 2023-10-09T04:09:00.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: debug 2023-10-09T04:09:00.214+0000 7f1123600700 1 -- 172.21.15.82:0/3027508915 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561f039be4e0 con 0x561f03ab2c00 2023-10-09T04:09:00.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: debug 2023-10-09T04:09:00.370+0000 7f11337c8680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:09:00.482 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: ignoring --setuser ceph since I am not root 2023-10-09T04:09:00.482 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: ignoring --setgroup ceph since I am not root 2023-10-09T04:09:00.482 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: debug 2023-10-09T04:09:00.214+0000 7ff8cfb6f700 1 -- 172.21.15.153:0/779854188 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5638d76f44e0 con 0x5638d77e8800 2023-10-09T04:09:00.483 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: debug 2023-10-09T04:09:00.370+0000 7ff8dfd37680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:09:00.870 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: debug 2023-10-09T04:09:00.478+0000 7ff8dfd37680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:09:00.972 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: debug 2023-10-09T04:09:00.478+0000 7f11337c8680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:09:01.169 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:00 smithi153 bash[28717]: debug 2023-10-09T04:09:00.970+0000 7ff8dfd37680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:09:01.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:01 smithi082 bash[33901]: audit 2023-10-09T04:09:00.172036+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:09:01.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:01 smithi082 bash[33901]: cluster 2023-10-09T04:09:00.172179+0000 mon.a (mon.0) 542 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T04:09:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:01 smithi082 bash[27462]: audit 2023-10-09T04:09:00.172036+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:09:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:01 smithi082 bash[27462]: cluster 2023-10-09T04:09:00.172179+0000 mon.a (mon.0) 542 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T04:09:01.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:00 smithi082 bash[27740]: debug 2023-10-09T04:09:00.966+0000 7f11337c8680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:09:01.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:01 smithi153 bash[27868]: audit 2023-10-09T04:09:00.172036+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.82:0/1653096734' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:09:01.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:01 smithi153 bash[27868]: cluster 2023-10-09T04:09:00.172179+0000 mon.a (mon.0) 542 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T04:09:02.175 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:01 smithi153 bash[28717]: debug 2023-10-09T04:09:01.918+0000 7ff8dfd37680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:09:02.175 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:02 smithi153 bash[28717]: debug 2023-10-09T04:09:02.022+0000 7ff8dfd37680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:09:02.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:01 smithi082 bash[27740]: debug 2023-10-09T04:09:01.902+0000 7f11337c8680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:09:02.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:02 smithi082 bash[27740]: debug 2023-10-09T04:09:01.998+0000 7f11337c8680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:09:02.675 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:02 smithi153 bash[28717]: debug 2023-10-09T04:09:02.294+0000 7ff8dfd37680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:09:02.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:02 smithi082 bash[27740]: debug 2023-10-09T04:09:02.254+0000 7f11337c8680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:09:03.667 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:03 smithi082 bash[27740]: debug 2023-10-09T04:09:03.398+0000 7f11337c8680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:09:03.668 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:03 smithi082 bash[27740]: debug 2023-10-09T04:09:03.562+0000 7f11337c8680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:09:03.708 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:03 smithi153 bash[28717]: debug 2023-10-09T04:09:03.442+0000 7ff8dfd37680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:09:03.708 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:03 smithi153 bash[28717]: debug 2023-10-09T04:09:03.602+0000 7ff8dfd37680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:09:03.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:03 smithi082 bash[27740]: debug 2023-10-09T04:09:03.658+0000 7f11337c8680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:09:03.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:03 smithi082 bash[27740]: debug 2023-10-09T04:09:03.774+0000 7f11337c8680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:09:04.046 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:03 smithi153 bash[28717]: debug 2023-10-09T04:09:03.702+0000 7ff8dfd37680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:09:04.046 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:03 smithi153 bash[28717]: debug 2023-10-09T04:09:03.814+0000 7ff8dfd37680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:09:04.367 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:03 smithi082 bash[27740]: debug 2023-10-09T04:09:03.994+0000 7f11337c8680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:09:04.415 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:04 smithi153 bash[28717]: debug 2023-10-09T04:09:04.042+0000 7ff8dfd37680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:09:04.675 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:04 smithi153 bash[28717]: debug 2023-10-09T04:09:04.410+0000 7ff8dfd37680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:09:04.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:04 smithi153 bash[28717]: debug 2023-10-09T04:09:04.534+0000 7ff8dfd37680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:09:04.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:04 smithi082 bash[27740]: debug 2023-10-09T04:09:04.362+0000 7f11337c8680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:09:04.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:04 smithi082 bash[27740]: debug 2023-10-09T04:09:04.482+0000 7f11337c8680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:09:05.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:05 smithi153 bash[28717]: debug 2023-10-09T04:09:05.398+0000 7ff8dfd37680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:09:05.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:05 smithi153 bash[28717]: debug 2023-10-09T04:09:05.502+0000 7ff8dfd37680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:09:05.677 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:05 smithi153 bash[28717]: debug 2023-10-09T04:09:05.614+0000 7ff8dfd37680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:09:05.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:05 smithi082 bash[27740]: debug 2023-10-09T04:09:05.354+0000 7f11337c8680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:09:05.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:05 smithi082 bash[27740]: debug 2023-10-09T04:09:05.466+0000 7f11337c8680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:09:05.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:05 smithi082 bash[27740]: debug 2023-10-09T04:09:05.582+0000 7f11337c8680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:09:06.102 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:05 smithi082 bash[27740]: debug 2023-10-09T04:09:05.806+0000 7f11337c8680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:09:06.138 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:05 smithi153 bash[28717]: debug 2023-10-09T04:09:05.838+0000 7ff8dfd37680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:09:06.425 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: debug 2023-10-09T04:09:06.134+0000 7ff8dfd37680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:09:06.425 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: debug 2023-10-09T04:09:06.298+0000 7ff8dfd37680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:09:06.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27740]: debug 2023-10-09T04:09:06.098+0000 7f11337c8680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:09:06.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27740]: debug 2023-10-09T04:09:06.262+0000 7f11337c8680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:09:06.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27740]: debug 2023-10-09T04:09:06.446+0000 7f11337c8680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:09:06.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27740]: debug 2023-10-09T04:09:06.662+0000 7f11337c8680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:09:06.802 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: debug 2023-10-09T04:09:06.474+0000 7ff8dfd37680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:09:06.802 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: debug 2023-10-09T04:09:06.690+0000 7ff8dfd37680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:09:07.058 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.794939+0000 mon.a (mon.0) 543 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:09:07.058 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.795553+0000 mon.a (mon.0) 544 : cluster [INF] Activating manager daemon y 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.807538+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.807873+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0125277s), standbys: x 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.810854+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.811097+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.811305+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.812559+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.812724+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.812895+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.813140+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.813462+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:09:07.059 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.813788+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.814140+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.814427+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.814615+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.814854+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.815075+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.815241+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.815678+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.823117+0000 mon.a (mon.0) 563 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.823205+0000 mon.a (mon.0) 564 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.827411+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:09:07.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.828878+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:09:07.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.832678+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:09:07.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: audit 2023-10-09T04:09:06.833161+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:09:07.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27462]: cluster 2023-10-09T04:09:06.841235+0000 mon.a (mon.0) 565 : cluster [INF] Manager daemon y is now available 2023-10-09T04:09:07.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.794939+0000 mon.a (mon.0) 543 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.795553+0000 mon.a (mon.0) 544 : cluster [INF] Activating manager daemon y 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.807538+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.807873+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0125277s), standbys: x 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.810854+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.811097+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.811305+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.812559+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.812724+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.812895+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:09:07.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.813140+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.813462+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.813788+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.814140+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.814427+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.814615+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.814854+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.815075+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.815241+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.815678+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.823117+0000 mon.a (mon.0) 563 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.823205+0000 mon.a (mon.0) 564 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:09:07.063 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.827411+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:09:07.064 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.828878+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:09:07.064 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.832678+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:09:07.064 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: audit 2023-10-09T04:09:06.833161+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:09:07.064 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[33901]: cluster 2023-10-09T04:09:06.841235+0000 mon.a (mon.0) 565 : cluster [INF] Manager daemon y is now available 2023-10-09T04:09:07.064 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:06 smithi082 bash[27740]: debug 2023-10-09T04:09:06.770+0000 7f11337c8680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:09:07.065 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: [09/Oct/2023:04:09:07] ENGINE Bus STARTING 2023-10-09T04:09:07.069 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: debug 2023-10-09T04:09:06.798+0000 7ff8dfd37680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:09:07.070 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: [09/Oct/2023:04:09:06] ENGINE Bus STARTING 2023-10-09T04:09:07.070 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: CherryPy Checker: 2023-10-09T04:09:07.070 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: The Application mounted at '' has an empty config. 2023-10-09T04:09:07.070 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: [09/Oct/2023:04:09:06] ENGINE Serving on http://:::9283 2023-10-09T04:09:07.070 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[28717]: [09/Oct/2023:04:09:06] ENGINE Bus STARTED 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.794939+0000 mon.a (mon.0) 543 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.795553+0000 mon.a (mon.0) 544 : cluster [INF] Activating manager daemon y 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.807538+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.807873+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0125277s), standbys: x 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.810854+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.811097+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.811305+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.812559+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:09:07.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.812724+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.812895+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.813140+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.813462+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.813788+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.814140+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.814427+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.814615+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.814854+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.815075+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.815241+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:09:07.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.815678+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.823117+0000 mon.a (mon.0) 563 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.823205+0000 mon.a (mon.0) 564 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.827411+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.828878+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.832678+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: audit 2023-10-09T04:09:06.833161+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.153:0/2704832910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:09:07.073 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:06 smithi153 bash[27868]: cluster 2023-10-09T04:09:06.841235+0000 mon.a (mon.0) 565 : cluster [INF] Manager daemon y is now available 2023-10-09T04:09:07.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: Warning: Permanently added '172.21.15.82' (ECDSA) to the list of known hosts. 2023-10-09T04:09:07.481 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: Warning: Permanently added '172.21.15.153' (ECDSA) to the list of known hosts. 2023-10-09T04:09:07.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: CherryPy Checker: 2023-10-09T04:09:07.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: The Application mounted at '' has an empty config. 2023-10-09T04:09:07.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: [09/Oct/2023:04:09:07] ENGINE Serving on http://:::9283 2023-10-09T04:09:07.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27740]: [09/Oct/2023:04:09:07] ENGINE Bus STARTED 2023-10-09T04:09:07.831 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled node-exporter update... 2023-10-09T04:09:08.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:06.987053+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:06.990415+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:07.027722+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:07.065949+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:07.072212+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: cluster 2023-10-09T04:09:07.816687+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e17: y(active, since 1.02133s), standbys: x 2023-10-09T04:09:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:07 smithi153 bash[27868]: audit 2023-10-09T04:09:07.829497+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:06.987053+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:06.990415+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:07.027722+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:07.065949+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:07.072212+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: cluster 2023-10-09T04:09:07.816687+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e17: y(active, since 1.02133s), standbys: x 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[27462]: audit 2023-10-09T04:09:07.829497+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:06.987053+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:06.990415+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:07.027722+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:07.065949+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:07.072212+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: cluster 2023-10-09T04:09:07.816687+0000 mon.a (mon.0) 571 : cluster [DBG] mgrmap e17: y(active, since 1.02133s), standbys: x 2023-10-09T04:09:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:07 smithi082 bash[33901]: audit 2023-10-09T04:09:07.829497+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:08.578 DEBUG:teuthology.orchestra.run.smithi082:node-exporter.a> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T04:09:08.581 DEBUG:teuthology.orchestra.run.smithi153:node-exporter.b> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T04:09:08.584 INFO:tasks.cephadm:Adding alertmanager.a on smithi082 2023-10-09T04:09:08.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi082=a' 2023-10-09T04:09:08.624 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:09:08.784 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:09:09.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:08 smithi153 bash[27868]: cephadm 2023-10-09T04:09:07.823053+0000 mgr.y (mgr.14436) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi082=a;smithi153=b;count:2 2023-10-09T04:09:09.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:08 smithi082 bash[27462]: cephadm 2023-10-09T04:09:07.823053+0000 mgr.y (mgr.14436) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi082=a;smithi153=b;count:2 2023-10-09T04:09:09.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:08 smithi082 bash[33901]: cephadm 2023-10-09T04:09:07.823053+0000 mgr.y (mgr.14436) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi082=a;smithi153=b;count:2 2023-10-09T04:09:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:10 smithi153 bash[27868]: cluster 2023-10-09T04:09:08.814166+0000 mgr.y (mgr.14436) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:10 smithi153 bash[27868]: cluster 2023-10-09T04:09:09.824795+0000 mon.a (mon.0) 573 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T04:09:10.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:10 smithi082 bash[27462]: cluster 2023-10-09T04:09:08.814166+0000 mgr.y (mgr.14436) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:10.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:10 smithi082 bash[27462]: cluster 2023-10-09T04:09:09.824795+0000 mon.a (mon.0) 573 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T04:09:10.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:10 smithi082 bash[33901]: cluster 2023-10-09T04:09:08.814166+0000 mgr.y (mgr.14436) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:10.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:10 smithi082 bash[33901]: cluster 2023-10-09T04:09:09.824795+0000 mon.a (mon.0) 573 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-09T04:09:11.617 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled alertmanager update... 2023-10-09T04:09:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:12 smithi153 bash[27868]: cluster 2023-10-09T04:09:10.814714+0000 mgr.y (mgr.14436) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:12 smithi153 bash[27868]: audit 2023-10-09T04:09:11.615559+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:12.284 DEBUG:teuthology.orchestra.run.smithi082:alertmanager.a> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T04:09:12.287 INFO:tasks.cephadm:Adding grafana.a on smithi153 2023-10-09T04:09:12.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi153=a' 2023-10-09T04:09:12.295 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:12 smithi082 bash[33901]: cluster 2023-10-09T04:09:10.814714+0000 mgr.y (mgr.14436) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:12.295 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:12 smithi082 bash[33901]: audit 2023-10-09T04:09:11.615559+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:12.296 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:12 smithi082 bash[27462]: cluster 2023-10-09T04:09:10.814714+0000 mgr.y (mgr.14436) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:12.296 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:12 smithi082 bash[27462]: audit 2023-10-09T04:09:11.615559+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:12.439 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:09:13.336 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:13 smithi153 bash[27868]: audit 2023-10-09T04:09:11.609027+0000 mgr.y (mgr.14436) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi082=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:13.337 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:13 smithi153 bash[27868]: cephadm 2023-10-09T04:09:11.610967+0000 mgr.y (mgr.14436) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi082=a;count:1 2023-10-09T04:09:13.337 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:13 smithi153 bash[27868]: audit 2023-10-09T04:09:12.647747+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:13.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[27462]: audit 2023-10-09T04:09:11.609027+0000 mgr.y (mgr.14436) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi082=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:13.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[27462]: cephadm 2023-10-09T04:09:11.610967+0000 mgr.y (mgr.14436) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi082=a;count:1 2023-10-09T04:09:13.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[27462]: audit 2023-10-09T04:09:12.647747+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:13.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[33901]: audit 2023-10-09T04:09:11.609027+0000 mgr.y (mgr.14436) 6 : audit [DBG] from='client.24382 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi082=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:13.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[33901]: cephadm 2023-10-09T04:09:11.610967+0000 mgr.y (mgr.14436) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi082=a;count:1 2023-10-09T04:09:13.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:13 smithi082 bash[33901]: audit 2023-10-09T04:09:12.647747+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: cluster 2023-10-09T04:09:12.815271+0000 mgr.y (mgr.14436) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.186451+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.479597+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.482000+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.542572+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.579563+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[27462]: audit 2023-10-09T04:09:13.586190+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: cluster 2023-10-09T04:09:12.815271+0000 mgr.y (mgr.14436) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.186451+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.479597+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.482000+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.542572+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.579563+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:14 smithi082 bash[33901]: audit 2023-10-09T04:09:13.586190+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: cluster 2023-10-09T04:09:12.815271+0000 mgr.y (mgr.14436) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.186451+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.479597+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.482000+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.542572+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.579563+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:14 smithi153 bash[27868]: audit 2023-10-09T04:09:13.586190+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[27462]: cephadm 2023-10-09T04:09:13.484546+0000 mgr.y (mgr.14436) 9 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[27462]: cephadm 2023-10-09T04:09:13.484971+0000 mgr.y (mgr.14436) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[27462]: cephadm 2023-10-09T04:09:13.511640+0000 mgr.y (mgr.14436) 11 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[27462]: cephadm 2023-10-09T04:09:13.523941+0000 mgr.y (mgr.14436) 12 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[27462]: cephadm 2023-10-09T04:09:13.608375+0000 mgr.y (mgr.14436) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[33901]: cephadm 2023-10-09T04:09:13.484546+0000 mgr.y (mgr.14436) 9 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[33901]: cephadm 2023-10-09T04:09:13.484971+0000 mgr.y (mgr.14436) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[33901]: cephadm 2023-10-09T04:09:13.511640+0000 mgr.y (mgr.14436) 11 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[33901]: cephadm 2023-10-09T04:09:13.523941+0000 mgr.y (mgr.14436) 12 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:15 smithi082 bash[33901]: cephadm 2023-10-09T04:09:13.608375+0000 mgr.y (mgr.14436) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2023-10-09T04:09:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:15 smithi153 bash[27868]: cephadm 2023-10-09T04:09:13.484546+0000 mgr.y (mgr.14436) 9 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:09:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:15 smithi153 bash[27868]: cephadm 2023-10-09T04:09:13.484971+0000 mgr.y (mgr.14436) 10 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:09:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:15 smithi153 bash[27868]: cephadm 2023-10-09T04:09:13.511640+0000 mgr.y (mgr.14436) 11 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:15 smithi153 bash[27868]: cephadm 2023-10-09T04:09:13.523941+0000 mgr.y (mgr.14436) 12 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:09:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:15 smithi153 bash[27868]: cephadm 2023-10-09T04:09:13.608375+0000 mgr.y (mgr.14436) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi153 2023-10-09T04:09:15.731 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled grafana update... 2023-10-09T04:09:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:16 smithi153 bash[27868]: cluster 2023-10-09T04:09:14.815901+0000 mgr.y (mgr.14436) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:16 smithi153 bash[27868]: audit 2023-10-09T04:09:15.729319+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:16 smithi082 bash[33901]: cluster 2023-10-09T04:09:14.815901+0000 mgr.y (mgr.14436) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:16 smithi082 bash[33901]: audit 2023-10-09T04:09:15.729319+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:16 smithi082 bash[27462]: cluster 2023-10-09T04:09:14.815901+0000 mgr.y (mgr.14436) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:16 smithi082 bash[27462]: audit 2023-10-09T04:09:15.729319+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:17.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:17 smithi082 bash[27462]: audit 2023-10-09T04:09:15.719692+0000 mgr.y (mgr.14436) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:17.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:17 smithi082 bash[27462]: cephadm 2023-10-09T04:09:15.723170+0000 mgr.y (mgr.14436) 16 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2023-10-09T04:09:17.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:17 smithi082 bash[33901]: audit 2023-10-09T04:09:15.719692+0000 mgr.y (mgr.14436) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:17.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:17 smithi082 bash[33901]: cephadm 2023-10-09T04:09:15.723170+0000 mgr.y (mgr.14436) 16 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2023-10-09T04:09:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:17 smithi153 bash[27868]: audit 2023-10-09T04:09:15.719692+0000 mgr.y (mgr.14436) 15 : audit [DBG] from='client.24388 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi153=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:17 smithi153 bash[27868]: cephadm 2023-10-09T04:09:15.723170+0000 mgr.y (mgr.14436) 16 : cephadm [INF] Saving service grafana spec with placement smithi153=a;count:1 2023-10-09T04:09:17.867 DEBUG:teuthology.orchestra.run.smithi153:grafana.a> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T04:09:17.869 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T04:09:17.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:09:18.067 INFO:journalctl@ceph.grafana.a.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:09:18.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:18 smithi082 bash[27462]: cluster 2023-10-09T04:09:16.816583+0000 mgr.y (mgr.14436) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:18.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:18 smithi082 bash[33901]: cluster 2023-10-09T04:09:16.816583+0000 mgr.y (mgr.14436) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:18 smithi153 bash[27868]: cluster 2023-10-09T04:09:16.816583+0000 mgr.y (mgr.14436) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:20 smithi082 bash[33901]: cluster 2023-10-09T04:09:18.817263+0000 mgr.y (mgr.14436) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:20 smithi082 bash[27462]: cluster 2023-10-09T04:09:18.817263+0000 mgr.y (mgr.14436) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:20 smithi153 bash[27868]: cluster 2023-10-09T04:09:18.817263+0000 mgr.y (mgr.14436) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:21.019 INFO:teuthology.orchestra.run.smithi082.stdout:[client.0] 2023-10-09T04:09:21.019 INFO:teuthology.orchestra.run.smithi082.stdout: key = AQDxfCNlh0rBABAAfzWcgA7LwoT2iU6MdhX+cA== 2023-10-09T04:09:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:21 smithi082 bash[33901]: audit 2023-10-09T04:09:21.012361+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:21.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:21 smithi082 bash[33901]: audit 2023-10-09T04:09:21.018206+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:21.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:21 smithi082 bash[27462]: audit 2023-10-09T04:09:21.012361+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:21.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:21 smithi082 bash[27462]: audit 2023-10-09T04:09:21.018206+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:21.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:21 smithi153 bash[27868]: audit 2023-10-09T04:09:21.012361+0000 mon.a (mon.0) 583 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:21.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:21 smithi153 bash[27868]: audit 2023-10-09T04:09:21.018206+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.82:0/3059573147' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:21.717 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:09:21.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T04:09:21.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T04:09:21.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:09:22.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:22 smithi082 bash[27462]: cluster 2023-10-09T04:09:20.817787+0000 mgr.y (mgr.14436) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:22.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:22 smithi082 bash[33901]: cluster 2023-10-09T04:09:20.817787+0000 mgr.y (mgr.14436) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:22.536 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:22 smithi153 bash[27868]: cluster 2023-10-09T04:09:20.817787+0000 mgr.y (mgr.14436) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:24 smithi153 bash[27868]: cluster 2023-10-09T04:09:22.818378+0000 mgr.y (mgr.14436) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:24 smithi082 bash[27462]: cluster 2023-10-09T04:09:22.818378+0000 mgr.y (mgr.14436) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:24 smithi082 bash[33901]: cluster 2023-10-09T04:09:22.818378+0000 mgr.y (mgr.14436) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:25.550 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:25 smithi153 bash[27868]: cluster 2023-10-09T04:09:24.819036+0000 mgr.y (mgr.14436) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:25 smithi082 bash[27462]: cluster 2023-10-09T04:09:24.819036+0000 mgr.y (mgr.14436) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:25.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:25 smithi082 bash[33901]: cluster 2023-10-09T04:09:24.819036+0000 mgr.y (mgr.14436) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:26.742 INFO:teuthology.orchestra.run.smithi153.stdout:[client.1] 2023-10-09T04:09:26.743 INFO:teuthology.orchestra.run.smithi153.stdout: key = AQD2fCNlFnXGKxAAXZ72jd7gP/hwPYMklhXOUw== 2023-10-09T04:09:26.785 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:26 smithi153 systemd[1]: Started Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:09:27.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:26 smithi153 bash[27868]: audit 2023-10-09T04:09:26.732905+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.153:0/1609638725' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:26 smithi153 bash[27868]: audit 2023-10-09T04:09:26.734087+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:26 smithi153 bash[27868]: audit 2023-10-09T04:09:26.741082+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:27.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:26 smithi153 bash[27868]: audit 2023-10-09T04:09:26.771386+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:27.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[33901]: audit 2023-10-09T04:09:26.732905+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.153:0/1609638725' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[33901]: audit 2023-10-09T04:09:26.734087+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[33901]: audit 2023-10-09T04:09:26.741082+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[33901]: audit 2023-10-09T04:09:26.771386+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[27462]: audit 2023-10-09T04:09:26.732905+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.153:0/1609638725' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[27462]: audit 2023-10-09T04:09:26.734087+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[27462]: audit 2023-10-09T04:09:26.741082+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:09:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:26 smithi082 bash[27462]: audit 2023-10-09T04:09:26.771386+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:27.544 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:27 smithi082 systemd[1]: Started Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:09:27.949 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2023-10-09T04:09:27.949 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T04:09:27.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T04:09:27.970 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T04:09:27.970 DEBUG:teuthology.orchestra.run.smithi082:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T04:09:27.981 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:27 smithi082 bash[52514]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T04:09:28.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[27868]: cephadm 2023-10-09T04:09:26.774167+0000 mgr.y (mgr.14436) 22 : cephadm [INF] Deploying daemon node-exporter.a on smithi082 2023-10-09T04:09:28.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[27868]: cluster 2023-10-09T04:09:26.819583+0000 mgr.y (mgr.14436) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[27868]: audit 2023-10-09T04:09:27.039341+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[27868]: audit 2023-10-09T04:09:27.523121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.203 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.166Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T04:09:28.203 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.167Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.167Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi153 (none))" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.167Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.167Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.171Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.171Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.173Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.174Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.174Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.619µs 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.174Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.174Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.175Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.074µs wal_replay_duration=285.564µs total_replay_duration=321.114µs 2023-10-09T04:09:28.204 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.176Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:09:28.205 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.176Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T04:09:28.205 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.176Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:09:28.205 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.201Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.469161ms db_storage=709ns remote_storage=1.427µs web_handler=268ns query_engine=695ns scrape=246.323µs scrape_sd=17.634µs notify=976ns notify_sd=1.568µs rules=24.968839ms 2023-10-09T04:09:28.205 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47071]: ts=2023-10-09T04:09:28.201Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T04:09:28.259 INFO:teuthology.misc.health.smithi082.stdout: 2023-10-09T04:09:28.259 INFO:teuthology.misc.health.smithi082.stdout:{"epoch":51,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:09:06.795028+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:09:28.272 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T04:09:28.272 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T04:09:28.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T04:09:28.323 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T04:09:28.323 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[27462]: cephadm 2023-10-09T04:09:26.774167+0000 mgr.y (mgr.14436) 22 : cephadm [INF] Deploying daemon node-exporter.a on smithi082 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[27462]: cluster 2023-10-09T04:09:26.819583+0000 mgr.y (mgr.14436) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[27462]: audit 2023-10-09T04:09:27.039341+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[27462]: audit 2023-10-09T04:09:27.523121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[33901]: cephadm 2023-10-09T04:09:26.774167+0000 mgr.y (mgr.14436) 22 : cephadm [INF] Deploying daemon node-exporter.a on smithi082 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[33901]: cluster 2023-10-09T04:09:26.819583+0000 mgr.y (mgr.14436) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[33901]: audit 2023-10-09T04:09:27.039341+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.324 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[33901]: audit 2023-10-09T04:09:27.523121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:28.586 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: aa2a8d90b84c: Pulling fs layer 2023-10-09T04:09:28.586 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: b45d31ee2d7f: Pulling fs layer 2023-10-09T04:09:28.587 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: b5db1e299295: Pulling fs layer 2023-10-09T04:09:28.675 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:28 smithi153 systemd[1]: Started Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:09:28.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47257]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T04:09:28.981 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: b45d31ee2d7f: Download complete 2023-10-09T04:09:28.982 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: aa2a8d90b84c: Verifying Checksum 2023-10-09T04:09:28.982 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: aa2a8d90b84c: Download complete 2023-10-09T04:09:28.982 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: b5db1e299295: Verifying Checksum 2023-10-09T04:09:28.982 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:28 smithi082 bash[52514]: b5db1e299295: Download complete 2023-10-09T04:09:29.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[27868]: cephadm 2023-10-09T04:09:27.524587+0000 mgr.y (mgr.14436) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2023-10-09T04:09:29.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[27868]: audit 2023-10-09T04:09:28.256884+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:29.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[27868]: audit 2023-10-09T04:09:28.257748+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.82:0/3031047901' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:09:29.176 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:28 smithi153 bash[47257]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T04:09:29.176 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: aa2a8d90b84c: Pulling fs layer 2023-10-09T04:09:29.176 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b45d31ee2d7f: Pulling fs layer 2023-10-09T04:09:29.176 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b5db1e299295: Pulling fs layer 2023-10-09T04:09:29.519 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[27462]: cephadm 2023-10-09T04:09:27.524587+0000 mgr.y (mgr.14436) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2023-10-09T04:09:29.519 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[27462]: audit 2023-10-09T04:09:28.256884+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:29.519 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[27462]: audit 2023-10-09T04:09:28.257748+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.82:0/3031047901' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:09:29.519 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[33901]: cephadm 2023-10-09T04:09:27.524587+0000 mgr.y (mgr.14436) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi153 2023-10-09T04:09:29.520 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[33901]: audit 2023-10-09T04:09:28.256884+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:29.520 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[33901]: audit 2023-10-09T04:09:28.257748+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.82:0/3031047901' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:09:29.675 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: aa2a8d90b84c: Verifying Checksum 2023-10-09T04:09:29.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: aa2a8d90b84c: Download complete 2023-10-09T04:09:29.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b45d31ee2d7f: Download complete 2023-10-09T04:09:29.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b5db1e299295: Verifying Checksum 2023-10-09T04:09:29.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b5db1e299295: Download complete 2023-10-09T04:09:29.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: aa2a8d90b84c: Pull complete 2023-10-09T04:09:29.981 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:29 smithi082 bash[52514]: aa2a8d90b84c: Pull complete 2023-10-09T04:09:30.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:30 smithi153 bash[27868]: cephadm 2023-10-09T04:09:28.266267+0000 mgr.y (mgr.14436) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi082 2023-10-09T04:09:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:30 smithi153 bash[27868]: cluster 2023-10-09T04:09:28.820239+0000 mgr.y (mgr.14436) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:30.176 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:29 smithi153 bash[47257]: b45d31ee2d7f: Pull complete 2023-10-09T04:09:30.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[27462]: cephadm 2023-10-09T04:09:28.266267+0000 mgr.y (mgr.14436) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi082 2023-10-09T04:09:30.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[27462]: cluster 2023-10-09T04:09:28.820239+0000 mgr.y (mgr.14436) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:30.325 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[33901]: cephadm 2023-10-09T04:09:28.266267+0000 mgr.y (mgr.14436) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi082 2023-10-09T04:09:30.325 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[33901]: cluster 2023-10-09T04:09:28.820239+0000 mgr.y (mgr.14436) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:30.675 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:30 smithi153 bash[47257]: b5db1e299295: Pull complete 2023-10-09T04:09:30.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:30 smithi153 bash[47257]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T04:09:30.676 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:30 smithi153 bash[47257]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T04:09:30.731 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[52514]: b45d31ee2d7f: Pull complete 2023-10-09T04:09:31.231 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:30 smithi082 bash[52514]: b5db1e299295: Pull complete 2023-10-09T04:09:31.926 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.614Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.614Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.615Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.615Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T04:09:31.927 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T04:09:31.928 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T04:09:31.929 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T04:09:31.930 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.616Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T04:09:31.931 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:09:31 smithi153 bash[47257]: ts=2023-10-09T04:09:31.617Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T04:09:32.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:32 smithi082 bash[27462]: cluster 2023-10-09T04:09:30.820599+0000 mgr.y (mgr.14436) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:32.216 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:31 smithi082 bash[52514]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T04:09:32.217 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:32 smithi082 bash[52514]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T04:09:32.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:32 smithi082 bash[33901]: cluster 2023-10-09T04:09:30.820599+0000 mgr.y (mgr.14436) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:32 smithi153 bash[27868]: cluster 2023-10-09T04:09:30.820599+0000 mgr.y (mgr.14436) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:33.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:33 smithi153 bash[27868]: audit 2023-10-09T04:09:32.045842+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:33.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:33 smithi082 bash[27462]: audit 2023-10-09T04:09:32.045842+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:33.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:33 smithi082 bash[33901]: audit 2023-10-09T04:09:32.045842+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:34.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[33901]: cluster 2023-10-09T04:09:32.820942+0000 mgr.y (mgr.14436) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:34.352 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[27462]: cluster 2023-10-09T04:09:32.820942+0000 mgr.y (mgr.14436) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:34.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:34 smithi153 bash[27868]: cluster 2023-10-09T04:09:32.820942+0000 mgr.y (mgr.14436) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:34.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:09:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:09:34.699 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.351Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.351Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T04:09:34.700 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.352Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T04:09:34.701 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T04:09:34.702 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T04:09:34.703 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T04:09:34.704 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:09:34 smithi082 bash[52514]: ts=2023-10-09T04:09:34.353Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T04:09:35.300 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'rbd' created 2023-10-09T04:09:35.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:35 smithi082 bash[27462]: audit 2023-10-09T04:09:34.791288+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:09:35.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:35 smithi082 bash[27462]: cluster 2023-10-09T04:09:34.821250+0000 mgr.y (mgr.14436) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:35.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:35 smithi082 bash[33901]: audit 2023-10-09T04:09:34.791288+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:09:35.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:35 smithi082 bash[33901]: cluster 2023-10-09T04:09:34.821250+0000 mgr.y (mgr.14436) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:35 smithi153 bash[27868]: audit 2023-10-09T04:09:34.791288+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:09:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:35 smithi153 bash[27868]: cluster 2023-10-09T04:09:34.821250+0000 mgr.y (mgr.14436) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:36.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T04:09:36.331 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[27462]: audit 2023-10-09T04:09:35.293697+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:09:36.332 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[27462]: cluster 2023-10-09T04:09:35.293884+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T04:09:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:36 smithi153 bash[27868]: audit 2023-10-09T04:09:35.293697+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:09:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:36 smithi153 bash[27868]: cluster 2023-10-09T04:09:35.293884+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T04:09:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:36 smithi153 bash[27868]: audit 2023-10-09T04:09:35.294688+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:09:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:36 smithi153 bash[27868]: audit 2023-10-09T04:09:35.298440+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:09:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[27462]: audit 2023-10-09T04:09:35.294688+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:09:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[27462]: audit 2023-10-09T04:09:35.298440+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:09:36.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[33901]: audit 2023-10-09T04:09:35.293697+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.82:0/628228711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:09:36.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[33901]: cluster 2023-10-09T04:09:35.293884+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T04:09:36.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[33901]: audit 2023-10-09T04:09:35.294688+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:09:36.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:36 smithi082 bash[33901]: audit 2023-10-09T04:09:35.298440+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T04:09:37.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:37 smithi153 bash[27868]: audit 2023-10-09T04:09:36.305353+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:09:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:37 smithi153 bash[27868]: cluster 2023-10-09T04:09:36.305562+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T04:09:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:37 smithi153 bash[27868]: cluster 2023-10-09T04:09:36.821724+0000 mgr.y (mgr.14436) 30 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:37.765 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[33901]: audit 2023-10-09T04:09:36.305353+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:09:37.766 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[33901]: cluster 2023-10-09T04:09:36.305562+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T04:09:37.766 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[33901]: cluster 2023-10-09T04:09:36.821724+0000 mgr.y (mgr.14436) 30 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:37.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[27462]: audit 2023-10-09T04:09:36.305353+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T04:09:37.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[27462]: cluster 2023-10-09T04:09:36.305562+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T04:09:37.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:37 smithi082 bash[27462]: cluster 2023-10-09T04:09:36.821724+0000 mgr.y (mgr.14436) 30 : cluster [DBG] pgmap v20: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:38.597 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:38 smithi082 bash[33901]: cluster 2023-10-09T04:09:37.322656+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:09:38.597 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:38 smithi082 bash[27462]: cluster 2023-10-09T04:09:37.322656+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:09:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:38 smithi153 bash[27868]: cluster 2023-10-09T04:09:37.322656+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:09:40.377 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:40 smithi082 bash[33901]: cluster 2023-10-09T04:09:38.822384+0000 mgr.y (mgr.14436) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:40.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:40 smithi082 bash[27462]: cluster 2023-10-09T04:09:38.822384+0000 mgr.y (mgr.14436) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:40 smithi153 bash[27868]: cluster 2023-10-09T04:09:38.822384+0000 mgr.y (mgr.14436) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:41.683 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T04:09:41.903 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[53213]: level=info ts=2023-10-09T04:09:41.775Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T04:09:41.903 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[27462]: audit 2023-10-09T04:09:40.664235+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:41.903 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[27462]: audit 2023-10-09T04:09:40.666304+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:41.903 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[27462]: cluster 2023-10-09T04:09:40.822761+0000 mgr.y (mgr.14436) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:41.903 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[27462]: audit 2023-10-09T04:09:40.983802+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.82:0/589805533' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:41.903 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[27462]: audit 2023-10-09T04:09:40.984286+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:41.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[33901]: audit 2023-10-09T04:09:40.664235+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:41.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[33901]: audit 2023-10-09T04:09:40.666304+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:41.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[33901]: cluster 2023-10-09T04:09:40.822761+0000 mgr.y (mgr.14436) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:41.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[33901]: audit 2023-10-09T04:09:40.983802+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.82:0/589805533' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:41.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:41 smithi082 bash[33901]: audit 2023-10-09T04:09:40.984286+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:41 smithi153 bash[27868]: audit 2023-10-09T04:09:40.664235+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:41 smithi153 bash[27868]: audit 2023-10-09T04:09:40.666304+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:41 smithi153 bash[27868]: cluster 2023-10-09T04:09:40.822761+0000 mgr.y (mgr.14436) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:41 smithi153 bash[27868]: audit 2023-10-09T04:09:40.983802+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.82:0/589805533' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:41 smithi153 bash[27868]: audit 2023-10-09T04:09:40.984286+0000 mon.a (mon.0) 602 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:42.355 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:09:42.355 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:09:42.355 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[27462]: audit 2023-10-09T04:09:41.676240+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[27462]: cluster 2023-10-09T04:09:41.676403+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[27462]: audit 2023-10-09T04:09:42.060040+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[33901]: audit 2023-10-09T04:09:41.676240+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[33901]: cluster 2023-10-09T04:09:41.676403+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:09:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[33901]: audit 2023-10-09T04:09:42.060040+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:42.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:42 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:09:42] "GET /metrics HTTP/1.1" 200 155659 "" "Prometheus/2.33.4" 2023-10-09T04:09:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:42 smithi153 bash[27868]: audit 2023-10-09T04:09:41.676240+0000 mon.a (mon.0) 603 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:42 smithi153 bash[27868]: cluster 2023-10-09T04:09:41.676403+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:09:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:42 smithi153 bash[27868]: audit 2023-10-09T04:09:42.060040+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:43.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:43 smithi153 bash[27868]: cluster 2023-10-09T04:09:42.823388+0000 mgr.y (mgr.14436) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:43 smithi082 bash[27462]: cluster 2023-10-09T04:09:42.823388+0000 mgr.y (mgr.14436) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:43 smithi082 bash[33901]: cluster 2023-10-09T04:09:42.823388+0000 mgr.y (mgr.14436) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:43.982 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:09:43 smithi082 bash[53213]: level=info ts=2023-10-09T04:09:43.727Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000867015s 2023-10-09T04:09:44.425 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:43 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:09:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:09:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:46 smithi153 bash[27868]: cluster 2023-10-09T04:09:44.824015+0000 mgr.y (mgr.14436) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:46.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[27462]: cluster 2023-10-09T04:09:44.824015+0000 mgr.y (mgr.14436) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[33901]: cluster 2023-10-09T04:09:44.824015+0000 mgr.y (mgr.14436) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:46.903 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[27740]: [09/Oct/2023:04:09:46] ENGINE Bus STOPPING 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[27740]: [09/Oct/2023:04:09:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[27740]: [09/Oct/2023:04:09:46] ENGINE Bus STOPPED 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:46 smithi082 bash[27740]: [09/Oct/2023:04:09:46] ENGINE Bus STARTING 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27740]: [09/Oct/2023:04:09:47] ENGINE Serving on http://:::9283 2023-10-09T04:09:47.234 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27740]: [09/Oct/2023:04:09:47] ENGINE Bus STARTED 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.317152+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.772116+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.773569+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.774703+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.780771+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: cluster 2023-10-09T04:09:46.824337+0000 mgr.y (mgr.14436) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.898755+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.903898+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.908032+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.909148+0000 mgr.y (mgr.14436) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:46.914381+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: cephadm 2023-10-09T04:09:46.923037+0000 mgr.y (mgr.14436) 37 : cephadm [INF] Deploying daemon grafana.a on smithi153 2023-10-09T04:09:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:47 smithi153 bash[27868]: audit 2023-10-09T04:09:47.069416+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.317152+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.772116+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.773569+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.774703+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.780771+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: cluster 2023-10-09T04:09:46.824337+0000 mgr.y (mgr.14436) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.898755+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.903898+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.908032+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.909148+0000 mgr.y (mgr.14436) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:46.914381+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: cephadm 2023-10-09T04:09:46.923037+0000 mgr.y (mgr.14436) 37 : cephadm [INF] Deploying daemon grafana.a on smithi153 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[27462]: audit 2023-10-09T04:09:47.069416+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.317152+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.772116+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.773569+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.774703+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.780771+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: cluster 2023-10-09T04:09:46.824337+0000 mgr.y (mgr.14436) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.898755+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.903898+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.908032+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.909148+0000 mgr.y (mgr.14436) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:46.914381+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: cephadm 2023-10-09T04:09:46.923037+0000 mgr.y (mgr.14436) 37 : cephadm [INF] Deploying daemon grafana.a on smithi153 2023-10-09T04:09:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:47 smithi082 bash[33901]: audit 2023-10-09T04:09:47.069416+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:09:49.964 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:09:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:50 smithi153 bash[27868]: cluster 2023-10-09T04:09:48.824945+0000 mgr.y (mgr.14436) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:50 smithi153 bash[27868]: audit 2023-10-09T04:09:49.954389+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.82:0/77878252' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:09:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:50 smithi082 bash[27462]: cluster 2023-10-09T04:09:48.824945+0000 mgr.y (mgr.14436) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:50 smithi082 bash[27462]: audit 2023-10-09T04:09:49.954389+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.82:0/77878252' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:09:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:50 smithi082 bash[33901]: cluster 2023-10-09T04:09:48.824945+0000 mgr.y (mgr.14436) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:50 smithi082 bash[33901]: audit 2023-10-09T04:09:49.954389+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.82:0/77878252' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:09:50.698 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":18,"active_gid":14436,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6800","nonce":1402928284},{"type":"v1","addr":"172.21.15.82:6801","nonce":1402928284}]},"active_addr":"172.21.15.82:6801/1402928284","active_change":"2023-10-09T04:09:06.795331+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.82:8443/","prometheus":"http://172.21.15.82:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":3169418397}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":590497495}]}]} 2023-10-09T04:09:50.703 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:09:50.703 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:09:50.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:09:52.033 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:09:51 smithi082 bash[53213]: level=info ts=2023-10-09T04:09:51.730Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003950711s 2023-10-09T04:09:52.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:52 smithi153 bash[27868]: cluster 2023-10-09T04:09:50.825505+0000 mgr.y (mgr.14436) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:52.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:52 smithi082 bash[27462]: cluster 2023-10-09T04:09:50.825505+0000 mgr.y (mgr.14436) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:52.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:52 smithi082 bash[33901]: cluster 2023-10-09T04:09:50.825505+0000 mgr.y (mgr.14436) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:53.231 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:09:52 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:09:52] "GET /metrics HTTP/1.1" 200 155664 "" "Prometheus/2.33.4" 2023-10-09T04:09:54.425 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:09:53 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:09:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:09:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:54 smithi153 bash[27868]: cluster 2023-10-09T04:09:52.826118+0000 mgr.y (mgr.14436) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:54 smithi082 bash[27462]: cluster 2023-10-09T04:09:52.826118+0000 mgr.y (mgr.14436) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:54 smithi082 bash[33901]: cluster 2023-10-09T04:09:52.826118+0000 mgr.y (mgr.14436) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:55.479 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:56.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:56 smithi082 bash[27462]: cluster 2023-10-09T04:09:54.826772+0000 mgr.y (mgr.14436) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:56.341 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:56 smithi082 bash[33901]: cluster 2023-10-09T04:09:54.826772+0000 mgr.y (mgr.14436) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:56.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:56 smithi153 bash[27868]: cluster 2023-10-09T04:09:54.826772+0000 mgr.y (mgr.14436) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:58.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:58 smithi082 bash[27462]: cluster 2023-10-09T04:09:56.827346+0000 mgr.y (mgr.14436) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:58 smithi082 bash[33901]: cluster 2023-10-09T04:09:56.827346+0000 mgr.y (mgr.14436) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:58.353 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:09:58.353 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":55,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:09:41.662556+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:09:58.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:58 smithi153 bash[27868]: cluster 2023-10-09T04:09:56.827346+0000 mgr.y (mgr.14436) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:59.070 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:09:59.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:09:59.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:09:59 smithi153 bash[27868]: audit 2023-10-09T04:09:58.350119+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.82:0/1719810540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:09:59.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:09:59 smithi082 bash[27462]: audit 2023-10-09T04:09:58.350119+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.82:0/1719810540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:09:59.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:09:59 smithi082 bash[33901]: audit 2023-10-09T04:09:58.350119+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.82:0/1719810540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:10:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:00 smithi153 bash[27868]: cluster 2023-10-09T04:09:58.828058+0000 mgr.y (mgr.14436) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:00 smithi153 bash[27868]: cluster 2023-10-09T04:10:00.000130+0000 mon.a (mon.0) 618 : cluster [INF] overall HEALTH_OK 2023-10-09T04:10:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:00 smithi082 bash[27462]: cluster 2023-10-09T04:09:58.828058+0000 mgr.y (mgr.14436) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:00 smithi082 bash[27462]: cluster 2023-10-09T04:10:00.000130+0000 mon.a (mon.0) 618 : cluster [INF] overall HEALTH_OK 2023-10-09T04:10:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:00 smithi082 bash[33901]: cluster 2023-10-09T04:09:58.828058+0000 mgr.y (mgr.14436) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:00 smithi082 bash[33901]: cluster 2023-10-09T04:10:00.000130+0000 mon.a (mon.0) 618 : cluster [INF] overall HEALTH_OK 2023-10-09T04:10:02.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:02 smithi153 bash[27868]: cluster 2023-10-09T04:10:00.828672+0000 mgr.y (mgr.14436) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:02.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:02 smithi082 bash[27462]: cluster 2023-10-09T04:10:00.828672+0000 mgr.y (mgr.14436) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:02.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:02 smithi082 bash[33901]: cluster 2023-10-09T04:10:00.828672+0000 mgr.y (mgr.14436) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:03.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:02] "GET /metrics HTTP/1.1" 200 155664 "" "Prometheus/2.33.4" 2023-10-09T04:10:03.837 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:04.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:04 smithi082 bash[27462]: cluster 2023-10-09T04:10:02.829267+0000 mgr.y (mgr.14436) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:04.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:04 smithi082 bash[33901]: cluster 2023-10-09T04:10:02.829267+0000 mgr.y (mgr.14436) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:04 smithi153 bash[27868]: cluster 2023-10-09T04:10:02.829267+0000 mgr.y (mgr.14436) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:04.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:10:03 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:10:06.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:06 smithi153 bash[27868]: cluster 2023-10-09T04:10:04.829803+0000 mgr.y (mgr.14436) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:06.389 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:06 smithi082 bash[33901]: cluster 2023-10-09T04:10:04.829803+0000 mgr.y (mgr.14436) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:06.390 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:06 smithi082 bash[27462]: cluster 2023-10-09T04:10:04.829803+0000 mgr.y (mgr.14436) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:06.816 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:10:06.816 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":55,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:09:41.662556+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:10:07.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:07 smithi082 bash[27462]: audit 2023-10-09T04:10:06.812787+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.82:0/2055383359' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:10:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:07 smithi082 bash[33901]: audit 2023-10-09T04:10:06.812787+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.82:0/2055383359' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:10:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:07 smithi153 bash[27868]: audit 2023-10-09T04:10:06.812787+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.82:0/2055383359' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:10:07.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:10:07.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:10:07.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:10:07.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:10:07.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:10:07.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:10:07.685 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:10:07.685 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:10:08.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:08 smithi153 bash[27868]: cluster 2023-10-09T04:10:06.830307+0000 mgr.y (mgr.14436) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:08.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:08 smithi082 bash[27462]: cluster 2023-10-09T04:10:06.830307+0000 mgr.y (mgr.14436) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:08.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:08 smithi082 bash[33901]: cluster 2023-10-09T04:10:06.830307+0000 mgr.y (mgr.14436) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:09.536 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[27868]: audit 2023-10-09T04:10:08.497499+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:09.537 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[27868]: audit 2023-10-09T04:10:08.499862+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:09.537 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[27868]: cluster 2023-10-09T04:10:08.831096+0000 mgr.y (mgr.14436) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:09.537 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T04:10:09.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T04:10:09.798 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T04:10:09.799 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[27462]: audit 2023-10-09T04:10:08.497499+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[27462]: audit 2023-10-09T04:10:08.499862+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[27462]: cluster 2023-10-09T04:10:08.831096+0000 mgr.y (mgr.14436) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[33901]: audit 2023-10-09T04:10:08.497499+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[33901]: audit 2023-10-09T04:10:08.499862+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:09 smithi082 bash[33901]: cluster 2023-10-09T04:10:08.831096+0000 mgr.y (mgr.14436) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:10.084 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T04:10:10.084 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T04:10:10.084 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T04:10:10.084 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T04:10:10.085 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:09 smithi153 bash[47868]: t=2023-10-09T04:10:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T04:10:10.085 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T04:10:10.085 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T04:10:10.085 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T04:10:10.336 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T04:10:10.336 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T04:10:10.337 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T04:10:10.589 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T04:10:10.841 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T04:10:10.841 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T04:10:10.841 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T04:10:10.842 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T04:10:10.842 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T04:10:10.842 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T04:10:10.842 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T04:10:10.842 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:10 smithi153 bash[47868]: t=2023-10-09T04:10:10+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T04:10:11.102 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T04:10:11.362 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T04:10:11.363 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T04:10:11.614 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T04:10:11.614 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T04:10:11.614 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T04:10:11.614 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T04:10:11.615 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T04:10:11.615 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T04:10:11.615 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T04:10:11.883 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T04:10:11.884 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T04:10:12.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[27868]: cluster 2023-10-09T04:10:10.831655+0000 mgr.y (mgr.14436) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:12.144 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T04:10:12.144 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:11 smithi153 bash[47868]: t=2023-10-09T04:10:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T04:10:12.145 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T04:10:12.145 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T04:10:12.145 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T04:10:12.145 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T04:10:12.145 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T04:10:12.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:11 smithi082 bash[27462]: cluster 2023-10-09T04:10:10.831655+0000 mgr.y (mgr.14436) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:11 smithi082 bash[33901]: cluster 2023-10-09T04:10:10.831655+0000 mgr.y (mgr.14436) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T04:10:12.414 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T04:10:12.614 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.615 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.615 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.621 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.621 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.621 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.622 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.622 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:12.674 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T04:10:12.674 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T04:10:12.674 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T04:10:12.674 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T04:10:12.674 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T04:10:12.675 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T04:10:12.675 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T04:10:12.675 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T04:10:12.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T04:10:12.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:12] "GET /metrics HTTP/1.1" 200 155671 "" "Prometheus/2.33.4" 2023-10-09T04:10:13.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[27868]: audit 2023-10-09T04:10:12.096201+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:12 smithi153 bash[47868]: t=2023-10-09T04:10:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T04:10:13.205 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T04:10:13.206 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T04:10:13.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:13 smithi082 bash[27462]: audit 2023-10-09T04:10:12.096201+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:13.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:13 smithi082 bash[33901]: audit 2023-10-09T04:10:12.096201+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T04:10:13.466 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T04:10:13.467 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T04:10:13.467 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T04:10:13.718 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T04:10:13.718 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T04:10:13.718 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T04:10:13.718 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T04:10:13.718 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T04:10:13.970 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T04:10:14.257 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[27868]: cluster 2023-10-09T04:10:12.832172+0000 mgr.y (mgr.14436) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:14.257 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[27868]: audit 2023-10-09T04:10:13.479266+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:14.258 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:10:14.258 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:13 smithi153 bash[47868]: t=2023-10-09T04:10:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T04:10:14.258 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T04:10:14.258 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T04:10:14.258 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T04:10:14.259 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T04:10:14.259 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T04:10:14.259 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T04:10:14.259 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T04:10:14.416 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:14 smithi082 bash[33901]: cluster 2023-10-09T04:10:12.832172+0000 mgr.y (mgr.14436) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:14.416 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:14 smithi082 bash[33901]: audit 2023-10-09T04:10:13.479266+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:14.417 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:14 smithi082 bash[27462]: cluster 2023-10-09T04:10:12.832172+0000 mgr.y (mgr.14436) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:14.417 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:14 smithi082 bash[27462]: audit 2023-10-09T04:10:13.479266+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:14.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T04:10:14.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T04:10:14.534 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T04:10:14.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T04:10:14.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T04:10:14.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T04:10:14.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T04:10:14.535 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T04:10:14.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T04:10:14.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T04:10:14.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T04:10:14.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T04:10:14.795 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T04:10:14.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T04:10:14.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T04:10:14.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T04:10:14.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T04:10:14.796 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T04:10:15.081 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T04:10:15.081 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T04:10:15.081 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T04:10:15.081 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T04:10:15.081 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T04:10:15.082 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T04:10:15.082 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:14 smithi153 bash[47868]: t=2023-10-09T04:10:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T04:10:15.082 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T04:10:15.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: audit 2023-10-09T04:10:14.255790+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.350 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: audit 2023-10-09T04:10:14.509209+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:15.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: audit 2023-10-09T04:10:14.510390+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:15.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: audit 2023-10-09T04:10:14.519021+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: cephadm 2023-10-09T04:10:14.522919+0000 mgr.y (mgr.14436) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:10:15.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: cephadm 2023-10-09T04:10:14.526198+0000 mgr.y (mgr.14436) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:10:15.351 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[27868]: cluster 2023-10-09T04:10:14.832511+0000 mgr.y (mgr.14436) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:15.351 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T04:10:15.351 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T04:10:15.352 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T04:10:15.602 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T04:10:15.602 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T04:10:15.602 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T04:10:15.603 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T04:10:15.603 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T04:10:15.603 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T04:10:15.603 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T04:10:15.603 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: audit 2023-10-09T04:10:14.255790+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: audit 2023-10-09T04:10:14.509209+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: audit 2023-10-09T04:10:14.510390+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: audit 2023-10-09T04:10:14.519021+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: cephadm 2023-10-09T04:10:14.522919+0000 mgr.y (mgr.14436) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: cephadm 2023-10-09T04:10:14.526198+0000 mgr.y (mgr.14436) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:10:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[27462]: cluster 2023-10-09T04:10:14.832511+0000 mgr.y (mgr.14436) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: audit 2023-10-09T04:10:14.255790+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: audit 2023-10-09T04:10:14.509209+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: audit 2023-10-09T04:10:14.510390+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: audit 2023-10-09T04:10:14.519021+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: cephadm 2023-10-09T04:10:14.522919+0000 mgr.y (mgr.14436) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: cephadm 2023-10-09T04:10:14.526198+0000 mgr.y (mgr.14436) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:10:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:15 smithi082 bash[33901]: cluster 2023-10-09T04:10:14.832511+0000 mgr.y (mgr.14436) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:15.871 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T04:10:15.872 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:15 smithi153 bash[47868]: t=2023-10-09T04:10:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T04:10:16.124 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T04:10:16.402 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T04:10:16.403 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T04:10:16.670 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T04:10:16.671 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T04:10:16.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T04:10:16.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T04:10:16.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T04:10:16.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T04:10:16.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T04:10:16.927 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T04:10:16.927 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T04:10:16.927 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:16 smithi153 bash[47868]: t=2023-10-09T04:10:16+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T04:10:17.201 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T04:10:17.202 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T04:10:17.202 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T04:10:17.486 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T04:10:17.486 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T04:10:17.486 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T04:10:17.486 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T04:10:17.487 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T04:10:17.487 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T04:10:17.487 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T04:10:17.487 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T04:10:17.487 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T04:10:17.772 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T04:10:17.772 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T04:10:17.773 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T04:10:18.037 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:17 smithi082 bash[27462]: cluster 2023-10-09T04:10:16.833094+0000 mgr.y (mgr.14436) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:18.037 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:17 smithi082 bash[33901]: cluster 2023-10-09T04:10:16.833094+0000 mgr.y (mgr.14436) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:18.058 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[27868]: cluster 2023-10-09T04:10:16.833094+0000 mgr.y (mgr.14436) 54 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:18.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T04:10:18.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T04:10:18.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T04:10:18.059 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:17 smithi153 bash[47868]: t=2023-10-09T04:10:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T04:10:18.060 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T04:10:18.328 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T04:10:18.328 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T04:10:18.328 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T04:10:18.328 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T04:10:18.328 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T04:10:18.329 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T04:10:18.329 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T04:10:18.329 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T04:10:18.582 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T04:10:18.583 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T04:10:18.852 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T04:10:18.852 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T04:10:18.852 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T04:10:18.853 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T04:10:19.121 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T04:10:19.121 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T04:10:19.121 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T04:10:19.121 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T04:10:19.122 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:18 smithi153 bash[47868]: t=2023-10-09T04:10:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T04:10:19.122 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T04:10:19.122 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T04:10:19.122 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T04:10:19.374 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T04:10:19.374 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T04:10:19.374 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T04:10:19.374 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T04:10:19.374 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T04:10:19.375 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T04:10:19.375 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T04:10:19.375 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T04:10:19.375 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T04:10:19.634 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T04:10:19.634 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T04:10:19.634 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T04:10:19.635 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T04:10:19.635 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T04:10:19.635 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T04:10:19.635 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T04:10:19.635 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T04:10:19.895 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T04:10:19.895 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T04:10:19.895 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T04:10:19.896 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T04:10:19.896 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T04:10:19.896 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T04:10:19.898 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T04:10:19.898 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T04:10:19.898 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T04:10:20.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:20 smithi082 bash[27462]: cluster 2023-10-09T04:10:18.833549+0000 mgr.y (mgr.14436) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:20.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[27868]: cluster 2023-10-09T04:10:18.833549+0000 mgr.y (mgr.14436) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:20.147 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:19 smithi153 bash[47868]: t=2023-10-09T04:10:19+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T04:10:20.148 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T04:10:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:20 smithi082 bash[33901]: cluster 2023-10-09T04:10:18.833549+0000 mgr.y (mgr.14436) 55 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:20.416 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T04:10:20.417 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T04:10:20.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T04:10:20.677 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T04:10:20.954 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T04:10:20.954 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T04:10:20.954 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T04:10:20.955 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T04:10:21.223 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:20 smithi153 bash[47868]: t=2023-10-09T04:10:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T04:10:21.224 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T04:10:21.476 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.038303704s 2023-10-09T04:10:21.727 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T04:10:21.727 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T04:10:21.727 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T04:10:21.728 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T04:10:21.728 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T04:10:21.728 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T04:10:22.043 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T04:10:22.043 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:10:22.043 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T04:10:22.043 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:10:22.043 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:10:21 smithi153 bash[47868]: t=2023-10-09T04:10:21+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T04:10:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:22 smithi153 bash[27868]: cluster 2023-10-09T04:10:20.834014+0000 mgr.y (mgr.14436) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:22.526 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:22 smithi082 bash[27462]: cluster 2023-10-09T04:10:20.834014+0000 mgr.y (mgr.14436) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:22.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:22 smithi082 bash[33901]: cluster 2023-10-09T04:10:20.834014+0000 mgr.y (mgr.14436) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:23.243 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:22] "GET /metrics HTTP/1.1" 200 155668 "" "Prometheus/2.33.4" 2023-10-09T04:10:24.121 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:24 smithi082 bash[27462]: cluster 2023-10-09T04:10:22.834413+0000 mgr.y (mgr.14436) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:24.121 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:24 smithi082 bash[33901]: cluster 2023-10-09T04:10:22.834413+0000 mgr.y (mgr.14436) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:24 smithi153 bash[27868]: cluster 2023-10-09T04:10:22.834413+0000 mgr.y (mgr.14436) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:24.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:10:23 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:10:26.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:26 smithi082 bash[27462]: cluster 2023-10-09T04:10:24.834766+0000 mgr.y (mgr.14436) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:26.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:26 smithi082 bash[33901]: cluster 2023-10-09T04:10:24.834766+0000 mgr.y (mgr.14436) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:26 smithi153 bash[27868]: cluster 2023-10-09T04:10:24.834766+0000 mgr.y (mgr.14436) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:28.303 INFO:teuthology.orchestra.run.smithi082.stdout:206158430231 2023-10-09T04:10:28.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:10:28.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:28 smithi082 bash[33901]: cluster 2023-10-09T04:10:26.835254+0000 mgr.y (mgr.14436) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:28.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:28 smithi082 bash[27462]: cluster 2023-10-09T04:10:26.835254+0000 mgr.y (mgr.14436) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:28 smithi153 bash[27868]: cluster 2023-10-09T04:10:26.835254+0000 mgr.y (mgr.14436) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:29.478 INFO:teuthology.orchestra.run.smithi082.stdout:34359738450 2023-10-09T04:10:29.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:10:29.729 INFO:teuthology.orchestra.run.smithi082.stdout:55834574922 2023-10-09T04:10:29.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:10:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:30 smithi153 bash[27868]: cluster 2023-10-09T04:10:28.835861+0000 mgr.y (mgr.14436) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:30.440 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:30 smithi082 bash[27462]: cluster 2023-10-09T04:10:28.835861+0000 mgr.y (mgr.14436) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:30.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:30 smithi082 bash[33901]: cluster 2023-10-09T04:10:28.835861+0000 mgr.y (mgr.14436) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:32.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:32 smithi153 bash[27868]: cluster 2023-10-09T04:10:30.836185+0000 mgr.y (mgr.14436) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:32.504 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:32 smithi082 bash[27462]: cluster 2023-10-09T04:10:30.836185+0000 mgr.y (mgr.14436) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:32.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:32 smithi082 bash[33901]: cluster 2023-10-09T04:10:30.836185+0000 mgr.y (mgr.14436) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:32.687 INFO:teuthology.orchestra.run.smithi082.stdout:77309411395 2023-10-09T04:10:32.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:10:32.728 INFO:teuthology.orchestra.run.smithi082.stdout:128849018929 2023-10-09T04:10:32.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:10:32.973 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:32] "GET /metrics HTTP/1.1" 200 155668 "" "Prometheus/2.33.4" 2023-10-09T04:10:33.409 INFO:teuthology.orchestra.run.smithi082.stdout:107374182456 2023-10-09T04:10:33.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:10:33.723 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:33 smithi082 systemd[1]: Stopping Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:10:33.723 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:33 smithi082 bash[56565]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T04:10:33.809 INFO:teuthology.orchestra.run.smithi082.stdout:180388626465 2023-10-09T04:10:33.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:10:33.850 INFO:teuthology.orchestra.run.smithi082.stdout:154618822696 2023-10-09T04:10:33.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:10:33.985 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:33 smithi082 bash[53213]: level=info ts=2023-10-09T04:10:33.721Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:10:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:34 smithi082 bash[33901]: cluster 2023-10-09T04:10:32.836570+0000 mgr.y (mgr.14436) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:34 smithi082 bash[27462]: cluster 2023-10-09T04:10:32.836570+0000 mgr.y (mgr.14436) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:34.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:34 smithi153 bash[27868]: cluster 2023-10-09T04:10:32.836570+0000 mgr.y (mgr.14436) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:34.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:10:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:10:34.427 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:34 smithi082 bash[56602]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:10:34.732 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:34 smithi082 bash[56778]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:10:34.732 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:34 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:10:34.732 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:34 smithi082 systemd[1]: Stopped Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:10:34.732 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:34 smithi082 systemd[1]: Started Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:10:35.725 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:35 smithi153 bash[27868]: audit 2023-10-09T04:10:34.465349+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:35.726 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:35 smithi153 bash[27868]: cephadm 2023-10-09T04:10:34.467451+0000 mgr.y (mgr.14436) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:10:35.726 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:35 smithi153 bash[27868]: cephadm 2023-10-09T04:10:34.472800+0000 mgr.y (mgr.14436) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:10:35.726 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:35 smithi153 bash[27868]: cluster 2023-10-09T04:10:34.837041+0000 mgr.y (mgr.14436) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[27462]: audit 2023-10-09T04:10:34.465349+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[27462]: cephadm 2023-10-09T04:10:34.467451+0000 mgr.y (mgr.14436) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[27462]: cephadm 2023-10-09T04:10:34.472800+0000 mgr.y (mgr.14436) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[27462]: cluster 2023-10-09T04:10:34.837041+0000 mgr.y (mgr.14436) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[33901]: audit 2023-10-09T04:10:34.465349+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[33901]: cephadm 2023-10-09T04:10:34.467451+0000 mgr.y (mgr.14436) 63 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:10:35.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[33901]: cephadm 2023-10-09T04:10:34.472800+0000 mgr.y (mgr.14436) 64 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:10:35.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[33901]: cluster 2023-10-09T04:10:34.837041+0000 mgr.y (mgr.14436) 65 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.477Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.478Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.480Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.82 port=9094 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.481Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.522Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.523Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:10:35.733 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.526Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T04:10:35.734 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:35 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:35.526Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T04:10:37.895 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:37 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:37.482Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000893409s 2023-10-09T04:10:37.972 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:37.973 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:37.978 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:37.984 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:37.985 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:37.990 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:37 smithi153 bash[27868]: cluster 2023-10-09T04:10:36.837521+0000 mgr.y (mgr.14436) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:38.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:37 smithi082 bash[27462]: cluster 2023-10-09T04:10:36.837521+0000 mgr.y (mgr.14436) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:38.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:37 smithi082 bash[33901]: cluster 2023-10-09T04:10:36.837521+0000 mgr.y (mgr.14436) 66 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:38.579 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 systemd[1]: Stopping Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[48454]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T04:10:38.580 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.389Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:10:38.581 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.390Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:10:38.581 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.390Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T04:10:38.581 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[47071]: ts=2023-10-09T04:10:38.390Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T04:10:39.115 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:38 smithi153 bash[48488]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:10:39.115 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:39 smithi153 bash[48552]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:10:39.115 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:39 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:10:39.115 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:39 smithi153 systemd[1]: Stopped Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:10:39.115 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:39 smithi153 systemd[1]: Started Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:10:39.451 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPING 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPED 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTING 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Serving on http://:::9283 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTED 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPING 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPED 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTING 2023-10-09T04:10:39.452 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Serving on http://:::9283 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTED 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPING 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STOPPED 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTING 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Serving on http://:::9283 2023-10-09T04:10:39.453 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:39 smithi082 bash[27740]: [09/Oct/2023:04:10:39] ENGINE Bus STARTED 2023-10-09T04:10:40.010 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:40.010 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.073Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.073Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.073Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi153 (none))" 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.073Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.073Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.075Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:10:40.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.076Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T04:10:40.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.077Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T04:10:40.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.081Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:10:40.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.081Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.444µs 2023-10-09T04:10:40.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.081Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: cluster 2023-10-09T04:10:38.838020+0000 mgr.y (mgr.14436) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.082070+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.087371+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.087921+0000 mgr.y (mgr.14436) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.089323+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.089774+0000 mgr.y (mgr.14436) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.098516+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.112256+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.113027+0000 mgr.y (mgr.14436) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.116144+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.116967+0000 mgr.y (mgr.14436) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.126793+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.135210+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.135914+0000 mgr.y (mgr.14436) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.140195+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.140657+0000 mgr.y (mgr.14436) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.148795+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[27868]: audit 2023-10-09T04:10:39.154814+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: cluster 2023-10-09T04:10:38.838020+0000 mgr.y (mgr.14436) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.082070+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.087371+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.087921+0000 mgr.y (mgr.14436) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.089323+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.089774+0000 mgr.y (mgr.14436) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.098516+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.112256+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.113027+0000 mgr.y (mgr.14436) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.116144+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.116967+0000 mgr.y (mgr.14436) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.126793+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.135210+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.135914+0000 mgr.y (mgr.14436) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.140195+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.140657+0000 mgr.y (mgr.14436) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.148795+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[27462]: audit 2023-10-09T04:10:39.154814+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: cluster 2023-10-09T04:10:38.838020+0000 mgr.y (mgr.14436) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.082070+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.087371+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.087921+0000 mgr.y (mgr.14436) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.089323+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.089774+0000 mgr.y (mgr.14436) 69 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi082.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.098516+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.112256+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.113027+0000 mgr.y (mgr.14436) 70 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.116144+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.116967+0000 mgr.y (mgr.14436) 71 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi153.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:10:40.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.126793+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.135210+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.135914+0000 mgr.y (mgr.14436) 72 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.140195+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.140657+0000 mgr.y (mgr.14436) 73 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi153.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.148795+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:40.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:40 smithi082 bash[33901]: audit 2023-10-09T04:10:39.154814+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:40.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.217Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T04:10:40.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.217Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.217Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=78.667µs wal_replay_duration=136.406971ms total_replay_duration=136.521413ms 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.221Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.221Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.221Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.246Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.941811ms db_storage=1.333µs remote_storage=2.372µs web_handler=712ns query_engine=1.184µs scrape=441.789µs scrape_sd=40.259µs notify=38.752µs notify_sd=13.211µs rules=23.818199ms 2023-10-09T04:10:40.677 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:10:40 smithi153 bash[48582]: ts=2023-10-09T04:10:40.247Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T04:10:42.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:42 smithi082 bash[27462]: cluster 2023-10-09T04:10:40.838420+0000 mgr.y (mgr.14436) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:42.348 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:42 smithi082 bash[33901]: cluster 2023-10-09T04:10:40.838420+0000 mgr.y (mgr.14436) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:42.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:42 smithi153 bash[27868]: cluster 2023-10-09T04:10:40.838420+0000 mgr.y (mgr.14436) 74 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:44.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:44 smithi082 bash[33901]: cluster 2023-10-09T04:10:42.838904+0000 mgr.y (mgr.14436) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:44.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:44 smithi082 bash[27462]: cluster 2023-10-09T04:10:42.838904+0000 mgr.y (mgr.14436) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:44 smithi153 bash[27868]: cluster 2023-10-09T04:10:42.838904+0000 mgr.y (mgr.14436) 75 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:45.982 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:10:45 smithi082 bash[56808]: level=info ts=2023-10-09T04:10:45.485Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003266503s 2023-10-09T04:10:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:46 smithi153 bash[27868]: cluster 2023-10-09T04:10:44.839244+0000 mgr.y (mgr.14436) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:46 smithi153 bash[27868]: audit 2023-10-09T04:10:45.002585+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:46.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:46 smithi082 bash[33901]: cluster 2023-10-09T04:10:44.839244+0000 mgr.y (mgr.14436) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:46.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:46 smithi082 bash[33901]: audit 2023-10-09T04:10:45.002585+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:46.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:46 smithi082 bash[27462]: cluster 2023-10-09T04:10:44.839244+0000 mgr.y (mgr.14436) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:46.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:46 smithi082 bash[27462]: audit 2023-10-09T04:10:45.002585+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:10:48.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:48 smithi082 bash[27462]: cluster 2023-10-09T04:10:46.839755+0000 mgr.y (mgr.14436) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:48.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:48 smithi082 bash[33901]: cluster 2023-10-09T04:10:46.839755+0000 mgr.y (mgr.14436) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:48.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:48 smithi153 bash[27868]: cluster 2023-10-09T04:10:46.839755+0000 mgr.y (mgr.14436) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:49.158 INFO:teuthology.orchestra.run.smithi082.stdout:128849018933 2023-10-09T04:10:49.227 INFO:teuthology.orchestra.run.smithi082.stdout:34359738455 2023-10-09T04:10:49.516 INFO:teuthology.orchestra.run.smithi082.stdout:206158430236 2023-10-09T04:10:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:50 smithi153 bash[27868]: cluster 2023-10-09T04:10:48.840211+0000 mgr.y (mgr.14436) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:50 smithi153 bash[27868]: audit 2023-10-09T04:10:49.157156+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.82:0/1486019291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:10:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:50 smithi153 bash[27868]: audit 2023-10-09T04:10:49.225485+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.82:0/2180978010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:10:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:50 smithi153 bash[27868]: audit 2023-10-09T04:10:49.515361+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.82:0/3216026083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[33901]: cluster 2023-10-09T04:10:48.840211+0000 mgr.y (mgr.14436) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[33901]: audit 2023-10-09T04:10:49.157156+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.82:0/1486019291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[33901]: audit 2023-10-09T04:10:49.225485+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.82:0/2180978010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[33901]: audit 2023-10-09T04:10:49.515361+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.82:0/3216026083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[27462]: cluster 2023-10-09T04:10:48.840211+0000 mgr.y (mgr.14436) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:50.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[27462]: audit 2023-10-09T04:10:49.157156+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.82:0/1486019291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:10:50.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[27462]: audit 2023-10-09T04:10:49.225485+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.82:0/2180978010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:10:50.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:50 smithi082 bash[27462]: audit 2023-10-09T04:10:49.515361+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.82:0/3216026083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:10:51.338 INFO:teuthology.orchestra.run.smithi082.stdout:77309411399 2023-10-09T04:10:52.122 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018929 got 128849018933 for osd.4 2023-10-09T04:10:52.123 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:52.345 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:52 smithi082 bash[27462]: cluster 2023-10-09T04:10:50.840788+0000 mgr.y (mgr.14436) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:52.345 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:52 smithi082 bash[27462]: audit 2023-10-09T04:10:51.337030+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.82:0/3794040085' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:10:52.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:52 smithi082 bash[33901]: cluster 2023-10-09T04:10:50.840788+0000 mgr.y (mgr.14436) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:52.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:52 smithi082 bash[33901]: audit 2023-10-09T04:10:51.337030+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.82:0/3794040085' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:10:52.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:52 smithi153 bash[27868]: cluster 2023-10-09T04:10:50.840788+0000 mgr.y (mgr.14436) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:52 smithi153 bash[27868]: audit 2023-10-09T04:10:51.337030+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.82:0/3794040085' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:10:52.593 INFO:teuthology.orchestra.run.smithi082.stdout:154618822700 2023-10-09T04:10:52.750 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738450 got 34359738455 for osd.0 2023-10-09T04:10:52.750 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:52.923 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:10:52 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:52] "GET /metrics HTTP/1.1" 200 155663 "" "Prometheus/2.33.4" 2023-10-09T04:10:53.092 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:53 smithi082 bash[27462]: audit 2023-10-09T04:10:52.591476+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.82:0/191746619' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:10:53.239 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:53 smithi082 bash[33901]: audit 2023-10-09T04:10:52.591476+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.82:0/191746619' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:10:53.298 INFO:teuthology.orchestra.run.smithi082.stdout:55834574928 2023-10-09T04:10:53.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:53 smithi153 bash[27868]: audit 2023-10-09T04:10:52.591476+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.82:0/191746619' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:10:53.470 INFO:teuthology.orchestra.run.smithi082.stdout:107374182461 2023-10-09T04:10:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[27462]: cluster 2023-10-09T04:10:52.841228+0000 mgr.y (mgr.14436) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[27462]: audit 2023-10-09T04:10:53.297157+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.82:0/2005866327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:10:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[27462]: audit 2023-10-09T04:10:53.466303+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.82:0/820316449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:10:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[33901]: cluster 2023-10-09T04:10:52.841228+0000 mgr.y (mgr.14436) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[33901]: audit 2023-10-09T04:10:53.297157+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.82:0/2005866327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:10:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:54 smithi082 bash[33901]: audit 2023-10-09T04:10:53.466303+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.82:0/820316449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:10:54.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:10:53 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:10:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:10:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:54 smithi153 bash[27868]: cluster 2023-10-09T04:10:52.841228+0000 mgr.y (mgr.14436) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:54 smithi153 bash[27868]: audit 2023-10-09T04:10:53.297157+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.82:0/2005866327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:10:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:54 smithi153 bash[27868]: audit 2023-10-09T04:10:53.466303+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.82:0/820316449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:10:54.609 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411395 got 77309411399 for osd.2 2023-10-09T04:10:54.610 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:54.656 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430231 got 206158430236 for osd.7 2023-10-09T04:10:54.656 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:54.664 INFO:teuthology.orchestra.run.smithi082.stdout:180388626470 2023-10-09T04:10:55.102 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822696 got 154618822700 for osd.5 2023-10-09T04:10:55.103 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:55 smithi153 bash[27868]: audit 2023-10-09T04:10:54.662814+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.82:0/993564791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:10:55.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:55 smithi082 bash[27462]: audit 2023-10-09T04:10:54.662814+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.82:0/993564791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:10:55.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:55 smithi082 bash[33901]: audit 2023-10-09T04:10:54.662814+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.82:0/993564791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:10:56.081 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574922 got 55834574928 for osd.1 2023-10-09T04:10:56.082 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:56.227 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182456 got 107374182461 for osd.3 2023-10-09T04:10:56.228 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:56.291 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626465 got 180388626470 for osd.6 2023-10-09T04:10:56.291 DEBUG:teuthology.parallel:result is None 2023-10-09T04:10:56.291 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:10:56.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:10:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:56 smithi153 bash[27868]: cluster 2023-10-09T04:10:54.841500+0000 mgr.y (mgr.14436) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:56 smithi082 bash[27462]: cluster 2023-10-09T04:10:54.841500+0000 mgr.y (mgr.14436) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:56.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:56 smithi082 bash[33901]: cluster 2023-10-09T04:10:54.841500+0000 mgr.y (mgr.14436) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:58.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:10:58 smithi153 bash[27868]: cluster 2023-10-09T04:10:56.842005+0000 mgr.y (mgr.14436) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:10:58 smithi082 bash[27462]: cluster 2023-10-09T04:10:56.842005+0000 mgr.y (mgr.14436) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:10:58.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:10:58 smithi082 bash[33901]: cluster 2023-10-09T04:10:56.842005+0000 mgr.y (mgr.14436) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: cluster 2023-10-09T04:10:58.842689+0000 mgr.y (mgr.14436) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: audit 2023-10-09T04:10:59.916366+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: audit 2023-10-09T04:10:59.917921+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: audit 2023-10-09T04:10:59.919381+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: audit 2023-10-09T04:10:59.925242+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:00 smithi153 bash[27868]: audit 2023-10-09T04:10:59.931993+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: cluster 2023-10-09T04:10:58.842689+0000 mgr.y (mgr.14436) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: audit 2023-10-09T04:10:59.916366+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: audit 2023-10-09T04:10:59.917921+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: audit 2023-10-09T04:10:59.919381+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: audit 2023-10-09T04:10:59.925242+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[27462]: audit 2023-10-09T04:10:59.931993+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: cluster 2023-10-09T04:10:58.842689+0000 mgr.y (mgr.14436) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: audit 2023-10-09T04:10:59.916366+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: audit 2023-10-09T04:10:59.917921+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: audit 2023-10-09T04:10:59.919381+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: audit 2023-10-09T04:10:59.925242+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:00 smithi082 bash[33901]: audit 2023-10-09T04:10:59.931993+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:01.066 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:02.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:02 smithi082 bash[27462]: cluster 2023-10-09T04:11:00.843251+0000 mgr.y (mgr.14436) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:02.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:02 smithi082 bash[33901]: cluster 2023-10-09T04:11:00.843251+0000 mgr.y (mgr.14436) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:02 smithi153 bash[27868]: cluster 2023-10-09T04:11:00.843251+0000 mgr.y (mgr.14436) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:03.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:02] "GET /metrics HTTP/1.1" 200 155663 "" "Prometheus/2.33.4" 2023-10-09T04:11:03.970 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:11:03.975 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:11:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:04 smithi153 bash[27868]: cluster 2023-10-09T04:11:02.843914+0000 mgr.y (mgr.14436) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:04.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:03 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:04 smithi082 bash[33901]: cluster 2023-10-09T04:11:02.843914+0000 mgr.y (mgr.14436) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:04 smithi082 bash[27462]: cluster 2023-10-09T04:11:02.843914+0000 mgr.y (mgr.14436) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:04.663 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":65,"stamp":"2023-10-09T04:11:02.843571+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380988,"kb_used_data":4220,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580228,"statfs":{"total":767960285184,"available":765522153472,"internally_reserved":0,"allocated":4321280,"data_stored":1763805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652302+0000","last_change":"2023-10-09T04:09:36.312935+0000","last_active":"2023-10-09T04:09:37.652302+0000","last_peered":"2023-10-09T04:09:37.652302+0000","last_clean":"2023-10-09T04:09:37.652302+0000","last_became_active":"2023-10-09T04:09:36.312042+0000","last_became_peered":"2023-10-09T04:09:36.312042+0000","last_unstale":"2023-10-09T04:09:37.652302+0000","last_undegraded":"2023-10-09T04:09:37.652302+0000","last_fullsized":"2023-10-09T04:09:37.652302+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.651731+0000","last_change":"2023-10-09T04:09:37.001556+0000","last_active":"2023-10-09T04:09:37.651731+0000","last_peered":"2023-10-09T04:09:37.651731+0000","last_clean":"2023-10-09T04:09:37.651731+0000","last_became_active":"2023-10-09T04:09:37.000571+0000","last_became_peered":"2023-10-09T04:09:37.000571+0000","last_unstale":"2023-10-09T04:09:37.651731+0000","last_undegraded":"2023-10-09T04:09:37.651731+0000","last_fullsized":"2023-10-09T04:09:37.651731+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652087+0000","last_change":"2023-10-09T04:09:37.118061+0000","last_active":"2023-10-09T04:09:37.652087+0000","last_peered":"2023-10-09T04:09:37.652087+0000","last_clean":"2023-10-09T04:09:37.652087+0000","last_became_active":"2023-10-09T04:09:37.117163+0000","last_became_peered":"2023-10-09T04:09:37.117163+0000","last_unstale":"2023-10-09T04:09:37.652087+0000","last_undegraded":"2023-10-09T04:09:37.652087+0000","last_fullsized":"2023-10-09T04:09:37.652087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652039+0000","last_change":"2023-10-09T04:09:37.117349+0000","last_active":"2023-10-09T04:09:37.652039+0000","last_peered":"2023-10-09T04:09:37.652039+0000","last_clean":"2023-10-09T04:09:37.652039+0000","last_became_active":"2023-10-09T04:09:37.116674+0000","last_became_peered":"2023-10-09T04:09:37.116674+0000","last_unstale":"2023-10-09T04:09:37.652039+0000","last_undegraded":"2023-10-09T04:09:37.652039+0000","last_fullsized":"2023-10-09T04:09:37.652039+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652310+0000","last_change":"2023-10-09T04:08:38.365338+0000","last_active":"2023-10-09T04:09:37.652310+0000","last_peered":"2023-10-09T04:09:37.652310+0000","last_clean":"2023-10-09T04:09:37.652310+0000","last_became_active":"2023-10-09T04:08:38.364495+0000","last_became_peered":"2023-10-09T04:08:38.364495+0000","last_unstale":"2023-10-09T04:09:37.652310+0000","last_undegraded":"2023-10-09T04:09:37.652310+0000","last_fullsized":"2023-10-09T04:09:37.652310+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.492868+0000","last_change":"2023-10-09T04:09:36.312522+0000","last_active":"2023-10-09T04:09:37.492868+0000","last_peered":"2023-10-09T04:09:37.492868+0000","last_clean":"2023-10-09T04:09:37.492868+0000","last_became_active":"2023-10-09T04:09:36.311979+0000","last_became_peered":"2023-10-09T04:09:36.311979+0000","last_unstale":"2023-10-09T04:09:37.492868+0000","last_undegraded":"2023-10-09T04:09:37.492868+0000","last_fullsized":"2023-10-09T04:09:37.492868+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.321599+0000","last_change":"2023-10-09T04:09:37.118566+0000","last_active":"2023-10-09T04:09:37.321599+0000","last_peered":"2023-10-09T04:09:37.321599+0000","last_clean":"2023-10-09T04:09:37.321599+0000","last_became_active":"2023-10-09T04:09:37.117693+0000","last_became_peered":"2023-10-09T04:09:37.117693+0000","last_unstale":"2023-10-09T04:09:37.321599+0000","last_undegraded":"2023-10-09T04:09:37.321599+0000","last_fullsized":"2023-10-09T04:09:37.321599+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.492815+0000","last_change":"2023-10-09T04:09:36.309784+0000","last_active":"2023-10-09T04:09:37.492815+0000","last_peered":"2023-10-09T04:09:37.492815+0000","last_clean":"2023-10-09T04:09:37.492815+0000","last_became_active":"2023-10-09T04:09:36.309394+0000","last_became_peered":"2023-10-09T04:09:36.309394+0000","last_unstale":"2023-10-09T04:09:37.492815+0000","last_undegraded":"2023-10-09T04:09:37.492815+0000","last_fullsized":"2023-10-09T04:09:37.492815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.651466+0000","last_change":"2023-10-09T04:09:37.117854+0000","last_active":"2023-10-09T04:09:37.651466+0000","last_peered":"2023-10-09T04:09:37.651466+0000","last_clean":"2023-10-09T04:09:37.651466+0000","last_became_active":"2023-10-09T04:09:37.117379+0000","last_became_peered":"2023-10-09T04:09:37.117379+0000","last_unstale":"2023-10-09T04:09:37.651466+0000","last_undegraded":"2023-10-09T04:09:37.651466+0000","last_fullsized":"2023-10-09T04:09:37.651466+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.878,"15min":0.88100000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.91500000000000004,"15min":0.91700000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.258,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.873}]},{"osd":1,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83499999999999996,"15min":0.83399999999999996},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.093,"5min":1.117,"15min":1.117},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.93999999999999995,"15min":0.93799999999999994},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.304,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.98199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88400000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.028,"5min":1.425,"15min":1.425},"last":1.099},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.65300000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2110000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.05}]},{"osd":3,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.88500000000000001,"15min":0.88400000000000001},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1719999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":1.018},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.94399999999999995,"15min":0.93999999999999995},"min":{"1min":0.65200000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.165,"5min":2.165,"15min":2.165},"last":0.95499999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.86699999999999999,"15min":0.86099999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.85199999999999998,"15min":0.84099999999999997},"min":{"1min":0.54700000000000004,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.92700000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"min":{"1min":0.54000000000000004,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1910000000000001,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.143},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.93999999999999995,"15min":0.93799999999999994},"min":{"1min":0.59799999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.391,"5min":1.673,"15min":1.673},"last":1.169}]},{"osd":6,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.97399999999999998,"15min":0.96599999999999997},"min":{"1min":0.67400000000000004,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.49,"5min":1.591,"15min":1.591},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.099,"15min":1.1220000000000001},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2909999999999999,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":1.1879999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626472,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.82799999999999996,"15min":0.82899999999999996},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.89100000000000001,"15min":0.90300000000000002},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.22,"5min":1.296,"15min":1.296},"last":0.73599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.90600000000000003,"15min":0.93400000000000005},"min":{"1min":0.504,"5min":0.502,"15min":0.502},"max":{"1min":1.117,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.075},{"interface":"front","average":{"1min":0.876,"5min":0.93400000000000005,"15min":0.95699999999999996},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.133,"5min":1.228,"15min":1.228},"last":0.88900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88400000000000001,"15min":0.878},"min":{"1min":0.60099999999999998,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.194,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.89000000000000001,"15min":0.91000000000000003},"min":{"1min":0.59199999999999997,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2390000000000001,"5min":1.242,"15min":1.242},"last":1.0189999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.85099999999999998,"15min":0.873},"min":{"1min":0.54000000000000004,"5min":0.496,"15min":0.496},"max":{"1min":1.262,"5min":1.278,"15min":1.278},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96299999999999997,"15min":0.96099999999999997},"min":{"1min":0.60499999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.339,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.0489999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.79900000000000004,"15min":0.77800000000000002},"min":{"1min":0.622,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.84599999999999997,"15min":0.84799999999999998},"min":{"1min":0.66300000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.77800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.91100000000000003,"15min":0.875},"min":{"1min":0.77000000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.73499999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.373,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.95999999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.96899999999999997,"15min":0.96299999999999997},"min":{"1min":0.67300000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.25,"5min":1.329,"15min":1.329},"last":1.099},{"interface":"front","average":{"1min":1.073,"5min":1.0389999999999999,"15min":1.0329999999999999},"min":{"1min":0.77800000000000002,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2669999999999999,"5min":1.369,"15min":1.369},"last":1.139}]}]},{"osd":1,"up_from":13,"seq":55834574930,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.67300000000000004,"15min":0.56499999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.377},"max":{"1min":0.84699999999999998,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.70199999999999996,"15min":0.58599999999999997},"min":{"1min":0.36299999999999999,"5min":0.35899999999999999,"15min":0.34699999999999998},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.71399999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71199999999999997,"15min":0.64200000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.73899999999999999,"15min":0.65900000000000003},"min":{"1min":0.53500000000000003,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.998,"5min":1,"15min":1},"last":0.83999999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.81799999999999995,"15min":0.80100000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.085,"5min":1.151,"15min":1.151},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.85299999999999998,"15min":0.81699999999999995},"min":{"1min":0.55500000000000005,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.107,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.91600000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.91100000000000003,"15min":0.92100000000000004},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.997,"15min":0.98399999999999999},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.3109999999999999,"5min":1.641,"15min":1.641},"last":1.1539999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.018,"15min":1.022},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":1.081},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0049999999999999,"15min":1.0229999999999999},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.92900000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.0429999999999999,"15min":1.0389999999999999},"min":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.024},{"interface":"front","average":{"1min":1.103,"5min":1.0469999999999999,"15min":1.038},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":3.327,"5min":3.327,"15min":3.327},"last":0.89500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.034,"15min":1.0409999999999999},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.3779999999999999,"5min":1.379,"15min":1.379},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.1060000000000001,"15min":1.1200000000000001},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.3029999999999999,"5min":1.706,"15min":1.706},"last":1.117}]}]},{"osd":0,"up_from":8,"seq":34359738458,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.64500000000000002,"15min":0.57899999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":1.01,"15min":1.01},"last":0.875},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.70899999999999996,"15min":0.61399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.39500000000000002},"max":{"1min":0.96199999999999997,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.67500000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.73599999999999999,"15min":0.66000000000000003},"min":{"1min":0.45100000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.0069999999999999,"5min":1.444,"15min":1.444},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.74099999999999999,"15min":0.67200000000000004},"min":{"1min":0.45800000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.96899999999999997,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.77200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.748,"15min":0.73999999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.98499999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.82399999999999995,"15min":0.81200000000000006},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.123,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.0169999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.997,"15min":0.97999999999999998},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97699999999999998,"15min":0.95899999999999996},"min":{"1min":0.59199999999999997,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3240000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.99199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.048,"15min":1.0469999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.323,"5min":1.458,"15min":1.458},"last":1.071},{"interface":"front","average":{"1min":1.006,"5min":1.018,"15min":1.012},"min":{"1min":0.68300000000000005,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.2350000000000001,"5min":1.353,"15min":1.353},"last":0.94499999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0840000000000001,"15min":1.095},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.3520000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.04},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.171,"15min":1.179},"min":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.2190000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0880000000000001,"15min":1.0940000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3200000000000001,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.149,"15min":1.155},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.367,"5min":1.448,"15min":1.448},"last":1.153}]}]},{"osd":2,"up_from":18,"seq":77309411402,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.64700000000000002,"15min":0.59499999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.90200000000000002,"5min":1.008,"15min":1.008},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.70799999999999996,"15min":0.63600000000000001},"min":{"1min":0.39300000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 04:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.72399999999999998,"15min":0.65400000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.03,"5min":1.077,"15min":1.077},"last":0.753},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.72499999999999998,"15min":0.67200000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.79200000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.73899999999999999,"15min":0.76500000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.77800000000000002,"15min":0.79700000000000004},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89100000000000001,"5min":1.083,"15min":1.083},"last":0.91000000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.93600000000000005,"15min":0.92800000000000005},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.26},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.96499999999999997,"15min":0.94899999999999995},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":1.0149999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0169999999999999,"15min":1.02},"min":{"1min":0.72899999999999998,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.3}]},{"osd":6,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.038,"15min":1.036},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.002,"5min":1.0629999999999999,"15min":1.0860000000000001},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.97699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.1040000000000001,"15min":1.115},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.3,"5min":1.371,"15min":1.371},"last":1.242},{"interface":"front","average":{"1min":1.04,"5min":1.056,"15min":1.0589999999999999},"min":{"1min":0.71099999999999997,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.361,"5min":1.401,"15min":1.401},"last":1.349}]}]},{"osd":3,"up_from":25,"seq":107374182463,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.70699999999999996,"15min":0.69899999999999995},"min":{"1min":0.52800000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.752,"5min":0.74299999999999999,"15min":0.72899999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.434}]},{"osd":1,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.72499999999999998,"15min":0.71499999999999997},"min":{"1min":0.39700000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.1200000000000001,"5min":1.1200000000000001,"15min":1.1200000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.79900000000000004,"15min":0.78400000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.181,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.66400000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86399999999999999,"15min":0.84199999999999997},"min":{"1min":0.621,"5min":0.432,"15min":0.432},"max":{"1min":1.224,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.80500000000000005,"15min":0.79700000000000004},"min":{"1min":0.56299999999999994,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":0.63100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.93400000000000005,"15min":0.90400000000000003},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0389999999999999,"15min":1.03},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.4259999999999999,"5min":1.482,"15min":1.482},"last":0.93600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.004,"15min":1.026},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.101,"15min":1.097},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.4930000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.85499999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:10:00 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0649999999999999,"15min":1.0640000000000001},"min":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.1359999999999999,"15min":1.133},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.89200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1120000000000001,"15min":1.1200000000000001},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.143,"5min":1.123,"15min":1.119},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.879}]}]},{"osd":4,"up_from":30,"seq":128849018936,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.85299999999999998,"15min":0.83399999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.879,"15min":0.85299999999999998},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.2569999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.224}]},{"osd":1,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84199999999999997,"15min":0.83599999999999997},"min":{"1min":0.54700000000000004,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2030000000000001,"5min":1.266,"15min":1.266},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.87,"5min":0.85599999999999998,"15min":0.82799999999999996},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.5469999999999999,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.86199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.88300000000000001,"15min":0.86099999999999999},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.91000000000000003,"15min":0.88600000000000001},"min":{"1min":0.69399999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.0489999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.90300000000000002,"15min":0.86899999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.89300000000000002,"15min":0.89200000000000002},"min":{"1min":0.61399999999999999,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.2310000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.90900000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.77500000000000002,"15min":0.754},"min":{"1min":0.51300000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.133,"5min":1.579,"15min":1.579},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.73399999999999999,"15min":0.69999999999999996},"min":{"1min":0.48399999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.018,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.79300000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.878,"15min":0.878},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.93999999999999995,"15min":0.93100000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2350000000000001,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.1459999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.1040000000000001,"15min":1.121},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.4359999999999999,"5min":1.653,"15min":1.653},"last":1.2609999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.046,"15min":1.0580000000000001},"min":{"1min":0.626,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":1.085}]}]},{"osd":5,"up_from":36,"seq":154618822703,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.82199999999999995,"15min":0.82799999999999996},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.84799999999999998,"15min":0.83899999999999997},"min":{"1min":0.66300000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":1.212}]},{"osd":1,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.82999999999999996,"15min":0.82699999999999996},"min":{"1min":0.56499999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.85499999999999998,"15min":0.84999999999999998},"min":{"1min":0.69999999999999996,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.80500000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.871,"15min":0.874},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.871,"15min":0.875},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.0980000000000001,"5min":1.161,"15min":1.161},"last":1.0229999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.86699999999999999,"15min":0.877},"min":{"1min":0.58799999999999997,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.91300000000000003,"15min":0.90900000000000003},"min":{"1min":0.60299999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.458,"5min":1.458,"15min":1.458},"last":0.95199999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.78200000000000003,"15min":0.76300000000000001},"min":{"1min":0.55700000000000005,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.84799999999999998,"15min":0.84999999999999998},"min":{"1min":0.61799999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.0369999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.81299999999999994,"15min":0.75700000000000001},"min":{"1min":0.71099999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.204,"5min":1.407,"15min":1.407},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.997,"5min":0.88300000000000001,"15min":0.86299999999999999},"min":{"1min":0.70799999999999996,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.2589999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.137}]},{"osd":7,"last update":"Mon Oct 9 04:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.90600000000000003,"15min":0.88300000000000001},"min":{"1min":0.76300000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.091,"5min":0.95199999999999996,"15min":0.92900000000000005},"min":{"1min":0.80300000000000005,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.6619999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.1659999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:11:04.666 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:11:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:05 smithi153 bash[27868]: audit 2023-10-09T04:11:03.958593+0000 mgr.y (mgr.14436) 86 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:05.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:05 smithi082 bash[33901]: audit 2023-10-09T04:11:03.958593+0000 mgr.y (mgr.14436) 86 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:05.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:05 smithi082 bash[27462]: audit 2023-10-09T04:11:03.958593+0000 mgr.y (mgr.14436) 86 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:06 smithi153 bash[27868]: cluster 2023-10-09T04:11:04.844387+0000 mgr.y (mgr.14436) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:06 smithi153 bash[27868]: audit 2023-10-09T04:11:05.794283+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:06 smithi153 bash[27868]: audit 2023-10-09T04:11:05.795585+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:06 smithi153 bash[27868]: audit 2023-10-09T04:11:05.797397+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:06 smithi153 bash[27868]: audit 2023-10-09T04:11:05.803369+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[27462]: cluster 2023-10-09T04:11:04.844387+0000 mgr.y (mgr.14436) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[27462]: audit 2023-10-09T04:11:05.794283+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[27462]: audit 2023-10-09T04:11:05.795585+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[27462]: audit 2023-10-09T04:11:05.797397+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[27462]: audit 2023-10-09T04:11:05.803369+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[33901]: cluster 2023-10-09T04:11:04.844387+0000 mgr.y (mgr.14436) 87 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[33901]: audit 2023-10-09T04:11:05.794283+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[33901]: audit 2023-10-09T04:11:05.795585+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[33901]: audit 2023-10-09T04:11:05.797397+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:06 smithi082 bash[33901]: audit 2023-10-09T04:11:05.803369+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:08 smithi153 bash[27868]: cluster 2023-10-09T04:11:06.844952+0000 mgr.y (mgr.14436) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:08.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:08 smithi082 bash[27462]: cluster 2023-10-09T04:11:06.844952+0000 mgr.y (mgr.14436) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:08.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:08 smithi082 bash[33901]: cluster 2023-10-09T04:11:06.844952+0000 mgr.y (mgr.14436) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:09.444 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:10.374 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:10 smithi082 bash[27462]: cluster 2023-10-09T04:11:08.845691+0000 mgr.y (mgr.14436) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:10.374 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:10 smithi082 bash[33901]: cluster 2023-10-09T04:11:08.845691+0000 mgr.y (mgr.14436) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:10 smithi153 bash[27868]: cluster 2023-10-09T04:11:08.845691+0000 mgr.y (mgr.14436) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:12.214 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:11:12.218 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:11:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:12 smithi153 bash[27868]: cluster 2023-10-09T04:11:10.846235+0000 mgr.y (mgr.14436) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:12.449 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:12 smithi082 bash[27462]: cluster 2023-10-09T04:11:10.846235+0000 mgr.y (mgr.14436) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:12.449 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:12 smithi082 bash[33901]: cluster 2023-10-09T04:11:10.846235+0000 mgr.y (mgr.14436) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:13.004 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-10-09T04:11:10.845973+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380988,"kb_used_data":4220,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580228,"statfs":{"total":767960285184,"available":765522153472,"internally_reserved":0,"allocated":4321280,"data_stored":1763805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652302+0000","last_change":"2023-10-09T04:09:36.312935+0000","last_active":"2023-10-09T04:09:37.652302+0000","last_peered":"2023-10-09T04:09:37.652302+0000","last_clean":"2023-10-09T04:09:37.652302+0000","last_became_active":"2023-10-09T04:09:36.312042+0000","last_became_peered":"2023-10-09T04:09:36.312042+0000","last_unstale":"2023-10-09T04:09:37.652302+0000","last_undegraded":"2023-10-09T04:09:37.652302+0000","last_fullsized":"2023-10-09T04:09:37.652302+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.651731+0000","last_change":"2023-10-09T04:09:37.001556+0000","last_active":"2023-10-09T04:09:37.651731+0000","last_peered":"2023-10-09T04:09:37.651731+0000","last_clean":"2023-10-09T04:09:37.651731+0000","last_became_active":"2023-10-09T04:09:37.000571+0000","last_became_peered":"2023-10-09T04:09:37.000571+0000","last_unstale":"2023-10-09T04:09:37.651731+0000","last_undegraded":"2023-10-09T04:09:37.651731+0000","last_fullsized":"2023-10-09T04:09:37.651731+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652087+0000","last_change":"2023-10-09T04:09:37.118061+0000","last_active":"2023-10-09T04:09:37.652087+0000","last_peered":"2023-10-09T04:09:37.652087+0000","last_clean":"2023-10-09T04:09:37.652087+0000","last_became_active":"2023-10-09T04:09:37.117163+0000","last_became_peered":"2023-10-09T04:09:37.117163+0000","last_unstale":"2023-10-09T04:09:37.652087+0000","last_undegraded":"2023-10-09T04:09:37.652087+0000","last_fullsized":"2023-10-09T04:09:37.652087+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652039+0000","last_change":"2023-10-09T04:09:37.117349+0000","last_active":"2023-10-09T04:09:37.652039+0000","last_peered":"2023-10-09T04:09:37.652039+0000","last_clean":"2023-10-09T04:09:37.652039+0000","last_became_active":"2023-10-09T04:09:37.116674+0000","last_became_peered":"2023-10-09T04:09:37.116674+0000","last_unstale":"2023-10-09T04:09:37.652039+0000","last_undegraded":"2023-10-09T04:09:37.652039+0000","last_fullsized":"2023-10-09T04:09:37.652039+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.652310+0000","last_change":"2023-10-09T04:08:38.365338+0000","last_active":"2023-10-09T04:09:37.652310+0000","last_peered":"2023-10-09T04:09:37.652310+0000","last_clean":"2023-10-09T04:09:37.652310+0000","last_became_active":"2023-10-09T04:08:38.364495+0000","last_became_peered":"2023-10-09T04:08:38.364495+0000","last_unstale":"2023-10-09T04:09:37.652310+0000","last_undegraded":"2023-10-09T04:09:37.652310+0000","last_fullsized":"2023-10-09T04:09:37.652310+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.492868+0000","last_change":"2023-10-09T04:09:36.312522+0000","last_active":"2023-10-09T04:09:37.492868+0000","last_peered":"2023-10-09T04:09:37.492868+0000","last_clean":"2023-10-09T04:09:37.492868+0000","last_became_active":"2023-10-09T04:09:36.311979+0000","last_became_peered":"2023-10-09T04:09:36.311979+0000","last_unstale":"2023-10-09T04:09:37.492868+0000","last_undegraded":"2023-10-09T04:09:37.492868+0000","last_fullsized":"2023-10-09T04:09:37.492868+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.321599+0000","last_change":"2023-10-09T04:09:37.118566+0000","last_active":"2023-10-09T04:09:37.321599+0000","last_peered":"2023-10-09T04:09:37.321599+0000","last_clean":"2023-10-09T04:09:37.321599+0000","last_became_active":"2023-10-09T04:09:37.117693+0000","last_became_peered":"2023-10-09T04:09:37.117693+0000","last_unstale":"2023-10-09T04:09:37.321599+0000","last_undegraded":"2023-10-09T04:09:37.321599+0000","last_fullsized":"2023-10-09T04:09:37.321599+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.492815+0000","last_change":"2023-10-09T04:09:36.309784+0000","last_active":"2023-10-09T04:09:37.492815+0000","last_peered":"2023-10-09T04:09:37.492815+0000","last_clean":"2023-10-09T04:09:37.492815+0000","last_became_active":"2023-10-09T04:09:36.309394+0000","last_became_peered":"2023-10-09T04:09:36.309394+0000","last_unstale":"2023-10-09T04:09:37.492815+0000","last_undegraded":"2023-10-09T04:09:37.492815+0000","last_fullsized":"2023-10-09T04:09:37.492815+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T04:09:37.651466+0000","last_change":"2023-10-09T04:09:37.117854+0000","last_active":"2023-10-09T04:09:37.651466+0000","last_peered":"2023-10-09T04:09:37.651466+0000","last_clean":"2023-10-09T04:09:37.651466+0000","last_became_active":"2023-10-09T04:09:37.117379+0000","last_became_peered":"2023-10-09T04:09:37.117379+0000","last_unstale":"2023-10-09T04:09:37.651466+0000","last_undegraded":"2023-10-09T04:09:37.651466+0000","last_fullsized":"2023-10-09T04:09:37.651466+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430240,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.878,"15min":0.88100000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.628},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.91500000000000004,"15min":0.91700000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.258,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.105}]},{"osd":1,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83499999999999996,"15min":0.83399999999999996},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.093,"5min":1.117,"15min":1.117},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.93999999999999995,"15min":0.93799999999999994},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.304,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.002}]},{"osd":2,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88400000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.028,"5min":1.425,"15min":1.425},"last":0.873},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.65300000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2110000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.80600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.88500000000000001,"15min":0.88400000000000001},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1719999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.94399999999999995,"15min":0.93999999999999995},"min":{"1min":0.65200000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.165,"5min":2.165,"15min":2.165},"last":1.0660000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.86699999999999999,"15min":0.86099999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.85199999999999998,"15min":0.84099999999999997},"min":{"1min":0.54700000000000004,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":1.1319999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"min":{"1min":0.54000000000000004,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1910000000000001,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.153},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.93999999999999995,"15min":0.93799999999999994},"min":{"1min":0.59799999999999998,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.391,"5min":1.673,"15min":1.673},"last":1.228}]},{"osd":6,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.97399999999999998,"15min":0.96599999999999997},"min":{"1min":0.67400000000000004,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.49,"5min":1.591,"15min":1.591},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.099,"15min":1.1220000000000001},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2909999999999999,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":1.2}]}]},{"osd":6,"up_from":42,"seq":180388626473,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.82799999999999996,"15min":0.82899999999999996},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.89100000000000001,"15min":0.90300000000000002},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.22,"5min":1.296,"15min":1.296},"last":1.1279999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.90600000000000003,"15min":0.93400000000000005},"min":{"1min":0.504,"5min":0.502,"15min":0.502},"max":{"1min":1.117,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.101},{"interface":"front","average":{"1min":0.876,"5min":0.93400000000000005,"15min":0.95699999999999996},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.133,"5min":1.228,"15min":1.228},"last":0.79000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88400000000000001,"15min":0.878},"min":{"1min":0.60099999999999998,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.194,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.89000000000000001,"15min":0.91000000000000003},"min":{"1min":0.59199999999999997,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2390000000000001,"5min":1.242,"15min":1.242},"last":1.016}]},{"osd":3,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.85099999999999998,"15min":0.873},"min":{"1min":0.54000000000000004,"5min":0.496,"15min":0.496},"max":{"1min":1.262,"5min":1.278,"15min":1.278},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96299999999999997,"15min":0.96099999999999997},"min":{"1min":0.60499999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.339,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.77000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.79900000000000004,"15min":0.77800000000000002},"min":{"1min":0.622,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.84599999999999997,"15min":0.84799999999999998},"min":{"1min":0.66300000000000003,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.97699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:00 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.91100000000000003,"15min":0.875},"min":{"1min":0.77000000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.874},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.73499999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.373,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.073}]},{"osd":7,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.96899999999999997,"15min":0.96299999999999997},"min":{"1min":0.67300000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.25,"5min":1.329,"15min":1.329},"last":1.002},{"interface":"front","average":{"1min":1.073,"5min":1.0389999999999999,"15min":1.0329999999999999},"min":{"1min":0.77800000000000002,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2669999999999999,"5min":1.369,"15min":1.369},"last":1.0489999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574931,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.67300000000000004,"15min":0.56499999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.377},"max":{"1min":0.84699999999999998,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.70199999999999996,"15min":0.58599999999999997},"min":{"1min":0.36299999999999999,"5min":0.35899999999999999,"15min":0.34699999999999998},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.77600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71199999999999997,"15min":0.64200000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.73899999999999999,"15min":0.65900000000000003},"min":{"1min":0.53500000000000003,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.998,"5min":1,"15min":1},"last":0.92600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.80600000000000005,"15min":0.79700000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0680000000000001,"5min":1.151,"15min":1.151},"last":1.048},{"interface":"front","average":{"1min":0.752,"5min":0.84299999999999997,"15min":0.81299999999999994},"min":{"1min":0.38900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":1.016}]},{"osd":4,"last update":"Mon Oct 9 04:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.91100000000000003,"15min":0.92100000000000004},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.997,"15min":0.98399999999999999},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.3109999999999999,"5min":1.641,"15min":1.641},"last":1.095}]},{"osd":5,"last update":"Mon Oct 9 04:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.018,"15min":1.022},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0049999999999999,"15min":1.0229999999999999},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.4330000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.028,"15min":1.034},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.268},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.0289999999999999,"15min":1.032},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.327,"5min":3.327,"15min":3.327},"last":1.1859999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.034,"15min":1.0409999999999999},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.3779999999999999,"5min":1.379,"15min":1.379},"last":1.502},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.1060000000000001,"15min":1.1200000000000001},"min":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.3029999999999999,"5min":1.706,"15min":1.706},"last":1.2390000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738459,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.64500000000000002,"15min":0.57899999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":1.01,"15min":1.01},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.70899999999999996,"15min":0.61399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.39500000000000002},"max":{"1min":0.96199999999999997,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.73599999999999999,"15min":0.66000000000000003},"min":{"1min":0.45100000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.0069999999999999,"5min":1.444,"15min":1.444},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.74099999999999999,"15min":0.67200000000000004},"min":{"1min":0.45800000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.96899999999999997,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.65600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.73099999999999998,"15min":0.73399999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.93000000000000005,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0269999999999999,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.63100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.997,"15min":0.97999999999999998},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.87},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97699999999999998,"15min":0.95899999999999996},"min":{"1min":0.59199999999999997,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3240000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.747}]},{"osd":5,"last update":"Mon Oct 9 04:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.048,"15min":1.0469999999999999},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.323,"5min":1.458,"15min":1.458},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.006,"5min":1.018,"15min":1.012},"min":{"1min":0.68300000000000005,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.2350000000000001,"5min":1.353,"15min":1.353},"last":0.81000000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:11:04 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.0669999999999999,"15min":1.089},"min":{"1min":0.70199999999999996,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1419999999999999,"15min":1.1699999999999999},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3320000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.99199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0880000000000001,"15min":1.0940000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3200000000000001,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.149,"15min":1.155},"min":{"1min":0.75,"5min":0.75,"15min":0.75},"max":{"1min":1.367,"5min":1.448,"15min":1.448},"last":1.0409999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411403,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.64700000000000002,"15min":0.59499999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.90200000000000002,"5min":1.008,"15min":1.008},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.70799999999999996,"15min":0.63600000000000001},"min":{"1min":0.39300000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.70099999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.72399999999999998,"15min":0.65400000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.03,"5min":1.077,"15min":1.077},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.72499999999999998,"15min":0.67200000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.73799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.73899999999999999,"15min":0.76500000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.77800000000000002,"15min":0.79700000000000004},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89100000000000001,"5min":1.083,"15min":1.083},"last":0.83299999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.93600000000000005,"15min":0.92800000000000005},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.96499999999999997,"15min":0.94899999999999995},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":1.254}]},{"osd":5,"last update":"Mon Oct 9 04:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0009999999999999,"15min":1.0069999999999999},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0169999999999999,"15min":1.02},"min":{"1min":0.72899999999999998,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.1060000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.038,"15min":1.036},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.087},{"interface":"front","average":{"1min":1.002,"5min":1.0629999999999999,"15min":1.0860000000000001},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.3660000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.1040000000000001,"15min":1.115},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.3,"5min":1.371,"15min":1.371},"last":1.2150000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.056,"15min":1.0589999999999999},"min":{"1min":0.71099999999999997,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.361,"5min":1.401,"15min":1.401},"last":1.179}]}]},{"osd":3,"up_from":25,"seq":107374182464,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.70699999999999996,"15min":0.69899999999999995},"min":{"1min":0.52800000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.752,"5min":0.74299999999999999,"15min":0.72899999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.65600000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.72499999999999998,"15min":0.71499999999999997},"min":{"1min":0.39700000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.1200000000000001,"5min":1.1200000000000001,"15min":1.1200000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.79900000000000004,"15min":0.78400000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.181,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.60899999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86399999999999999,"15min":0.84199999999999997},"min":{"1min":0.621,"5min":0.432,"15min":0.432},"max":{"1min":1.224,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.80500000000000005,"15min":0.79700000000000004},"min":{"1min":0.56299999999999994,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.421,"5min":1.421,"15min":1.421},"last":0.69799999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.93400000000000005,"15min":0.90400000000000003},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0389999999999999,"15min":1.03},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.4259999999999999,"5min":1.482,"15min":1.482},"last":0.97699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.004,"15min":1.026},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.101,"15min":1.097},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.4930000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.2170000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.054,"15min":1.0600000000000001},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":1.323},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.123,"15min":1.129},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.1220000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:10:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.1120000000000001,"15min":1.1200000000000001},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.143,"5min":1.123,"15min":1.119},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":1.264}]}]},{"osd":4,"up_from":30,"seq":128849018937,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.85299999999999998,"15min":0.83399999999999996},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.879,"15min":0.85299999999999998},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.2569999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.038}]},{"osd":1,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84199999999999997,"15min":0.83599999999999997},"min":{"1min":0.54700000000000004,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2030000000000001,"5min":1.266,"15min":1.266},"last":1.081},{"interface":"front","average":{"1min":0.87,"5min":0.85599999999999998,"15min":0.82799999999999996},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.5469999999999999,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.73699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.88300000000000001,"15min":0.86099999999999999},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.91000000000000003,"15min":0.88600000000000001},"min":{"1min":0.69399999999999995,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.1080000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.90300000000000002,"15min":0.86899999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.89300000000000002,"15min":0.89200000000000002},"min":{"1min":0.61399999999999999,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.2310000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.995}]},{"osd":5,"last update":"Mon Oct 9 04:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.77500000000000002,"15min":0.754},"min":{"1min":0.51300000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.133,"5min":1.579,"15min":1.579},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.73399999999999999,"15min":0.69999999999999996},"min":{"1min":0.48399999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.018,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.85199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.878,"15min":0.878},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.1259999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.93999999999999995,"15min":0.93100000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2350000000000001,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.90900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.1040000000000001,"15min":1.121},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.4359999999999999,"5min":1.653,"15min":1.653},"last":1.171},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.046,"15min":1.0580000000000001},"min":{"1min":0.626,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":1.0680000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822704,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.82199999999999995,"15min":0.82799999999999996},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.84799999999999998,"15min":0.83899999999999997},"min":{"1min":0.66300000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.97099999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.82999999999999996,"15min":0.82699999999999996},"min":{"1min":0.56499999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.85499999999999998,"15min":0.84999999999999998},"min":{"1min":0.69999999999999996,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.91700000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.871,"15min":0.874},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.871,"15min":0.875},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.0980000000000001,"5min":1.161,"15min":1.161},"last":0.85099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.86699999999999999,"15min":0.877},"min":{"1min":0.58799999999999997,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.91300000000000003,"15min":0.90900000000000003},"min":{"1min":0.60299999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.458,"5min":1.458,"15min":1.458},"last":1.0840000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.78200000000000003,"15min":0.76300000000000001},"min":{"1min":0.55700000000000005,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.84799999999999998,"15min":0.84999999999999998},"min":{"1min":0.61799999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":0.94799999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.81299999999999994,"15min":0.75700000000000001},"min":{"1min":0.71099999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.204,"5min":1.407,"15min":1.407},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.997,"5min":0.88300000000000001,"15min":0.86299999999999999},"min":{"1min":0.70799999999999996,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.2589999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.98799999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.90600000000000003,"15min":0.88300000000000001},"min":{"1min":0.76300000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.091,"5min":0.95199999999999996,"15min":0.92900000000000005},"min":{"1min":0.80300000000000005,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.6619999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.012}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:11:13.006 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:11:13.007 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:11:13.007 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:11:13.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:11:13.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:12] "GET /metrics HTTP/1.1" 200 155659 "" "Prometheus/2.33.4" 2023-10-09T04:11:13.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:13 smithi082 bash[33901]: audit 2023-10-09T04:11:12.202174+0000 mgr.y (mgr.14436) 91 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:13.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:13 smithi082 bash[27462]: audit 2023-10-09T04:11:12.202174+0000 mgr.y (mgr.14436) 91 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:13.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:13 smithi153 bash[27868]: audit 2023-10-09T04:11:12.202174+0000 mgr.y (mgr.14436) 91 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:14 smithi153 bash[27868]: cluster 2023-10-09T04:11:12.846858+0000 mgr.y (mgr.14436) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:14.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:13 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:14 smithi082 bash[33901]: cluster 2023-10-09T04:11:12.846858+0000 mgr.y (mgr.14436) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:14 smithi082 bash[27462]: cluster 2023-10-09T04:11:12.846858+0000 mgr.y (mgr.14436) 92 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:16 smithi153 bash[27868]: cluster 2023-10-09T04:11:14.847393+0000 mgr.y (mgr.14436) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:16 smithi082 bash[27462]: cluster 2023-10-09T04:11:14.847393+0000 mgr.y (mgr.14436) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:16.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:16 smithi082 bash[33901]: cluster 2023-10-09T04:11:14.847393+0000 mgr.y (mgr.14436) 93 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:17.780 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:18 smithi153 bash[27868]: cluster 2023-10-09T04:11:16.847941+0000 mgr.y (mgr.14436) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:18.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:18 smithi082 bash[33901]: cluster 2023-10-09T04:11:16.847941+0000 mgr.y (mgr.14436) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:18.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:18 smithi082 bash[27462]: cluster 2023-10-09T04:11:16.847941+0000 mgr.y (mgr.14436) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:20 smithi153 bash[27868]: cluster 2023-10-09T04:11:18.848394+0000 mgr.y (mgr.14436) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:20 smithi082 bash[33901]: cluster 2023-10-09T04:11:18.848394+0000 mgr.y (mgr.14436) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:20 smithi082 bash[27462]: cluster 2023-10-09T04:11:18.848394+0000 mgr.y (mgr.14436) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:20.588 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:11:21.315 INFO:teuthology.orchestra.run.smithi082.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:11:21.315 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:11:21.316 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T04:11:21.316 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T04:11:21.327 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi082.front.sepia.ceph.com 2023-10-09T04:11:21.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T04:11:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:21 smithi153 bash[27868]: audit 2023-10-09T04:11:20.586757+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.82:0/1608805189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:11:21.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:21 smithi082 bash[27462]: audit 2023-10-09T04:11:20.586757+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.82:0/1608805189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:11:21.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:21 smithi082 bash[33901]: audit 2023-10-09T04:11:20.586757+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.82:0/1608805189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:11:22.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:22 smithi153 bash[27868]: cluster 2023-10-09T04:11:20.848941+0000 mgr.y (mgr.14436) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:22.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:22 smithi082 bash[33901]: cluster 2023-10-09T04:11:20.848941+0000 mgr.y (mgr.14436) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:22.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:22 smithi082 bash[27462]: cluster 2023-10-09T04:11:20.848941+0000 mgr.y (mgr.14436) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:22.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:22] "GET /metrics HTTP/1.1" 200 155669 "" "Prometheus/2.33.4" 2023-10-09T04:11:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:24 smithi153 bash[27868]: cluster 2023-10-09T04:11:22.849522+0000 mgr.y (mgr.14436) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:24.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:23 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:24 smithi082 bash[27462]: cluster 2023-10-09T04:11:22.849522+0000 mgr.y (mgr.14436) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:24 smithi082 bash[33901]: cluster 2023-10-09T04:11:22.849522+0000 mgr.y (mgr.14436) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:25.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:25 smithi082 bash[27462]: audit 2023-10-09T04:11:24.460161+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:11:25.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:25 smithi082 bash[33901]: audit 2023-10-09T04:11:24.460161+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:11:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:25 smithi153 bash[27868]: audit 2023-10-09T04:11:24.460161+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: audit 2023-10-09T04:11:24.459182+0000 mgr.y (mgr.14436) 98 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: cluster 2023-10-09T04:11:24.850061+0000 mgr.y (mgr.14436) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: audit 2023-10-09T04:11:25.193047+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: cluster 2023-10-09T04:11:25.193134+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: audit 2023-10-09T04:11:25.202896+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[27462]: debug 2023-10-09T04:11:26.218+0000 7fdd2c501700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[33901]: audit 2023-10-09T04:11:24.459182+0000 mgr.y (mgr.14436) 98 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[33901]: cluster 2023-10-09T04:11:24.850061+0000 mgr.y (mgr.14436) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[33901]: audit 2023-10-09T04:11:25.193047+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:11:26.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[33901]: cluster 2023-10-09T04:11:25.193134+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:11:26.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:26 smithi082 bash[33901]: audit 2023-10-09T04:11:25.202896+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:26 smithi153 bash[27868]: audit 2023-10-09T04:11:24.459182+0000 mgr.y (mgr.14436) 98 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:11:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:26 smithi153 bash[27868]: cluster 2023-10-09T04:11:24.850061+0000 mgr.y (mgr.14436) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:26 smithi153 bash[27868]: audit 2023-10-09T04:11:25.193047+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:11:26.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:26 smithi153 bash[27868]: cluster 2023-10-09T04:11:25.193134+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:11:26.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:26 smithi153 bash[27868]: audit 2023-10-09T04:11:25.202896+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:27.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T04:11:27.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.200758+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.200942+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.222605+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.223922+0000 mon.a (mon.0) 660 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.223981+0000 mon.a (mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.233187+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.234086+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.234495+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:0 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: cephadm 2023-10-09T04:11:26.239075+0000 mgr.y (mgr.14436) 100 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.244416+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.273151+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.566379+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.567403+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.574371+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.583130+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.584105+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.588455+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[33901]: audit 2023-10-09T04:11:26.589372+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.200758+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.200942+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.222605+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.223922+0000 mon.a (mon.0) 660 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.223981+0000 mon.a (mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.233187+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.234086+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.234495+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:0 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: cephadm 2023-10-09T04:11:26.239075+0000 mgr.y (mgr.14436) 100 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.244416+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.273151+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.566379+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.567403+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.574371+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.583130+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.584105+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.588455+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:27.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:27 smithi082 bash[27462]: audit 2023-10-09T04:11:26.589372+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:27.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.200758+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.200942+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.222605+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.223922+0000 mon.a (mon.0) 660 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.223981+0000 mon.a (mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.233187+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.234086+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.234495+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:0 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: cephadm 2023-10-09T04:11:26.239075+0000 mgr.y (mgr.14436) 100 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T04:11:27.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.244416+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.273151+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.566379+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.567403+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.574371+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.583130+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:27.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.584105+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:27.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.588455+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:27.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:27 smithi153 bash[27868]: audit 2023-10-09T04:11:26.589372+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:28.545 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[27462]: cephadm 2023-10-09T04:11:26.590229+0000 mgr.y (mgr.14436) 101 : cephadm [INF] Deploying daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:11:28.545 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[27462]: cluster 2023-10-09T04:11:26.850702+0000 mgr.y (mgr.14436) 102 : cluster [DBG] pgmap v80: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:28.546 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[27462]: cluster 2023-10-09T04:11:27.234908+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:11:28.546 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[33901]: cephadm 2023-10-09T04:11:26.590229+0000 mgr.y (mgr.14436) 101 : cephadm [INF] Deploying daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:11:28.546 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[33901]: cluster 2023-10-09T04:11:26.850702+0000 mgr.y (mgr.14436) 102 : cluster [DBG] pgmap v80: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:28.546 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:28 smithi082 bash[33901]: cluster 2023-10-09T04:11:27.234908+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:11:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:28 smithi153 bash[27868]: cephadm 2023-10-09T04:11:26.590229+0000 mgr.y (mgr.14436) 101 : cephadm [INF] Deploying daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:11:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:28 smithi153 bash[27868]: cluster 2023-10-09T04:11:26.850702+0000 mgr.y (mgr.14436) 102 : cluster [DBG] pgmap v80: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:28 smithi153 bash[27868]: cluster 2023-10-09T04:11:27.234908+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:11:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:29 smithi153 bash[27868]: cluster 2023-10-09T04:11:28.237526+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:11:29.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:29 smithi082 bash[33901]: cluster 2023-10-09T04:11:28.237526+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:11:29.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:29 smithi082 bash[27462]: cluster 2023-10-09T04:11:28.237526+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:11:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[33901]: cluster 2023-10-09T04:11:28.851170+0000 mgr.y (mgr.14436) 103 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[33901]: audit 2023-10-09T04:11:30.209509+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[33901]: audit 2023-10-09T04:11:30.210967+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[33901]: audit 2023-10-09T04:11:30.214934+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[33901]: audit 2023-10-09T04:11:30.216039+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[27462]: cluster 2023-10-09T04:11:28.851170+0000 mgr.y (mgr.14436) 103 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[27462]: audit 2023-10-09T04:11:30.209509+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[27462]: audit 2023-10-09T04:11:30.210967+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[27462]: audit 2023-10-09T04:11:30.214934+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:30 smithi082 bash[27462]: audit 2023-10-09T04:11:30.216039+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:30 smithi153 bash[27868]: cluster 2023-10-09T04:11:28.851170+0000 mgr.y (mgr.14436) 103 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:30 smithi153 bash[27868]: audit 2023-10-09T04:11:30.209509+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:30 smithi153 bash[27868]: audit 2023-10-09T04:11:30.210967+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:11:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:30 smithi153 bash[27868]: audit 2023-10-09T04:11:30.214934+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:11:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:30 smithi153 bash[27868]: audit 2023-10-09T04:11:30.216039+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:31.523 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:31 smithi153 bash[27868]: cephadm 2023-10-09T04:11:30.217009+0000 mgr.y (mgr.14436) 104 : cephadm [INF] Deploying daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:11:31.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:31 smithi153 bash[27868]: cluster 2023-10-09T04:11:30.851768+0000 mgr.y (mgr.14436) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:31.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:31 smithi082 bash[33901]: cephadm 2023-10-09T04:11:30.217009+0000 mgr.y (mgr.14436) 104 : cephadm [INF] Deploying daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:11:31.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:31 smithi082 bash[33901]: cluster 2023-10-09T04:11:30.851768+0000 mgr.y (mgr.14436) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:31.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:31 smithi082 bash[27462]: cephadm 2023-10-09T04:11:30.217009+0000 mgr.y (mgr.14436) 104 : cephadm [INF] Deploying daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:11:31.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:31 smithi082 bash[27462]: cluster 2023-10-09T04:11:30.851768+0000 mgr.y (mgr.14436) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:32.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T04:11:33.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:32] "GET /metrics HTTP/1.1" 200 155669 "" "Prometheus/2.33.4" 2023-10-09T04:11:33.533 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.261690+0000 mon.a (mon.0) 680 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:boot 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.261745+0000 mon.a (mon.0) 681 : cluster [INF] daemon mds.foo.smithi082.juwitc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.261827+0000 mon.a (mon.0) 682 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.261846+0000 mon.a (mon.0) 683 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.261858+0000 mon.a (mon.0) 684 : cluster [INF] Cluster is now healthy 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.262093+0000 mon.a (mon.0) 685 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: audit 2023-10-09T04:11:32.262200+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:11:33.534 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.267813+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:creating} 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.286074+0000 mon.a (mon.0) 688 : cluster [INF] daemon mds.foo.smithi082.juwitc is now active in filesystem foo as rank 0 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: audit 2023-10-09T04:11:32.710199+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: audit 2023-10-09T04:11:32.712167+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[27462]: cluster 2023-10-09T04:11:32.852469+0000 mgr.y (mgr.14436) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.261690+0000 mon.a (mon.0) 680 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:boot 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.261745+0000 mon.a (mon.0) 681 : cluster [INF] daemon mds.foo.smithi082.juwitc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.261827+0000 mon.a (mon.0) 682 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.261846+0000 mon.a (mon.0) 683 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.261858+0000 mon.a (mon.0) 684 : cluster [INF] Cluster is now healthy 2023-10-09T04:11:33.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.262093+0000 mon.a (mon.0) 685 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: audit 2023-10-09T04:11:32.262200+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.267813+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:creating} 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.286074+0000 mon.a (mon.0) 688 : cluster [INF] daemon mds.foo.smithi082.juwitc is now active in filesystem foo as rank 0 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: audit 2023-10-09T04:11:32.710199+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: audit 2023-10-09T04:11:32.712167+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:33.536 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:33 smithi082 bash[33901]: cluster 2023-10-09T04:11:32.852469+0000 mgr.y (mgr.14436) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.261690+0000 mon.a (mon.0) 680 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:boot 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.261745+0000 mon.a (mon.0) 681 : cluster [INF] daemon mds.foo.smithi082.juwitc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.261827+0000 mon.a (mon.0) 682 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.261846+0000 mon.a (mon.0) 683 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.261858+0000 mon.a (mon.0) 684 : cluster [INF] Cluster is now healthy 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.262093+0000 mon.a (mon.0) 685 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: audit 2023-10-09T04:11:32.262200+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:11:33.572 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.267813+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:creating} 2023-10-09T04:11:33.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.286074+0000 mon.a (mon.0) 688 : cluster [INF] daemon mds.foo.smithi082.juwitc is now active in filesystem foo as rank 0 2023-10-09T04:11:33.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: audit 2023-10-09T04:11:32.710199+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:33.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: audit 2023-10-09T04:11:32.712167+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:33.573 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[27868]: cluster 2023-10-09T04:11:32.852469+0000 mgr.y (mgr.14436) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:34.280 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:34.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[27462]: cluster 2023-10-09T04:11:33.273490+0000 mon.a (mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:active 2023-10-09T04:11:34.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[27462]: cluster 2023-10-09T04:11:33.273557+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 2023-10-09T04:11:34.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[27462]: cluster 2023-10-09T04:11:34.060590+0000 mon.a (mon.0) 693 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] up:boot 2023-10-09T04:11:34.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[27462]: cluster 2023-10-09T04:11:34.060653+0000 mon.a (mon.0) 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[27462]: audit 2023-10-09T04:11:34.061122+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[33901]: cluster 2023-10-09T04:11:33.273490+0000 mon.a (mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:active 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[33901]: cluster 2023-10-09T04:11:33.273557+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[33901]: cluster 2023-10-09T04:11:34.060590+0000 mon.a (mon.0) 693 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] up:boot 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[33901]: cluster 2023-10-09T04:11:34.060653+0000 mon.a (mon.0) 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:11:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:34 smithi082 bash[33901]: audit 2023-10-09T04:11:34.061122+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:11:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:34 smithi153 bash[27868]: cluster 2023-10-09T04:11:33.273490+0000 mon.a (mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] up:active 2023-10-09T04:11:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:34 smithi153 bash[27868]: cluster 2023-10-09T04:11:33.273557+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 2023-10-09T04:11:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:34 smithi153 bash[27868]: cluster 2023-10-09T04:11:34.060590+0000 mon.a (mon.0) 693 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] up:boot 2023-10-09T04:11:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:34 smithi153 bash[27868]: cluster 2023-10-09T04:11:34.060653+0000 mon.a (mon.0) 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:11:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:34 smithi153 bash[27868]: audit 2023-10-09T04:11:34.061122+0000 mon.a (mon.0) 695 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:11:35.602 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:35 smithi082 bash[33901]: cluster 2023-10-09T04:11:34.853130+0000 mgr.y (mgr.14436) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:35.602 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:35 smithi082 bash[27462]: cluster 2023-10-09T04:11:34.853130+0000 mgr.y (mgr.14436) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:35 smithi153 bash[27868]: cluster 2023-10-09T04:11:34.853130+0000 mgr.y (mgr.14436) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:36.125 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:11:36.135 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:11:36.135 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:11:36.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:11:38.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:38 smithi153 bash[27868]: cluster 2023-10-09T04:11:36.853776+0000 mgr.y (mgr.14436) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:38.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:38 smithi153 bash[27868]: audit 2023-10-09T04:11:37.147754+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:38 smithi082 bash[27462]: cluster 2023-10-09T04:11:36.853776+0000 mgr.y (mgr.14436) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:38 smithi082 bash[27462]: audit 2023-10-09T04:11:37.147754+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:38.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:38 smithi082 bash[33901]: cluster 2023-10-09T04:11:36.853776+0000 mgr.y (mgr.14436) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:38.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:38 smithi082 bash[33901]: audit 2023-10-09T04:11:37.147754+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.400536+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.647315+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.648982+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.650470+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.657779+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.667209+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.669601+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.670904+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: audit 2023-10-09T04:11:38.679012+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:39 smithi153 bash[27868]: cluster 2023-10-09T04:11:38.854976+0000 mgr.y (mgr.14436) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:39.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.400536+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.647315+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.648982+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.650470+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.657779+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.667209+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.669601+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.670904+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: audit 2023-10-09T04:11:38.679012+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[27462]: cluster 2023-10-09T04:11:38.854976+0000 mgr.y (mgr.14436) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.400536+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.647315+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.648982+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.650470+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.657779+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.667209+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.669601+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.670904+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: audit 2023-10-09T04:11:38.679012+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:11:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:39 smithi082 bash[33901]: cluster 2023-10-09T04:11:38.854976+0000 mgr.y (mgr.14436) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:40.533 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:41 smithi153 bash[27868]: cluster 2023-10-09T04:11:40.855601+0000 mgr.y (mgr.14436) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:42.220 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:41 smithi082 bash[27462]: cluster 2023-10-09T04:11:40.855601+0000 mgr.y (mgr.14436) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:42.220 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:41 smithi082 bash[33901]: cluster 2023-10-09T04:11:40.855601+0000 mgr.y (mgr.14436) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:43.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:42 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:42] "GET /metrics HTTP/1.1" 200 191348 "" "Prometheus/2.33.4" 2023-10-09T04:11:43.476 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:11:43.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:43 smithi082 bash[27462]: cluster 2023-10-09T04:11:42.856329+0000 mgr.y (mgr.14436) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:43.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:43 smithi082 bash[27462]: audit 2023-10-09T04:11:43.464865+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.82:0/3310072556' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:11:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:43 smithi082 bash[33901]: cluster 2023-10-09T04:11:42.856329+0000 mgr.y (mgr.14436) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:43 smithi082 bash[33901]: audit 2023-10-09T04:11:43.464865+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.82:0/3310072556' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:11:44.123 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":18,"active_gid":14436,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6800","nonce":1402928284},{"type":"v1","addr":"172.21.15.82:6801","nonce":1402928284}]},"active_addr":"172.21.15.82:6801/1402928284","active_change":"2023-10-09T04:09:06.795331+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.82:8443/","prometheus":"http://172.21.15.82:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":3169418397}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":590497495}]}]} 2023-10-09T04:11:44.128 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:11:44.128 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:11:44.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:11:44.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:43 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:43 smithi153 bash[27868]: cluster 2023-10-09T04:11:42.856329+0000 mgr.y (mgr.14436) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:43 smithi153 bash[27868]: audit 2023-10-09T04:11:43.464865+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.82:0/3310072556' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:11:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:46 smithi153 bash[27868]: cluster 2023-10-09T04:11:44.856931+0000 mgr.y (mgr.14436) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:46 smithi082 bash[27462]: cluster 2023-10-09T04:11:44.856931+0000 mgr.y (mgr.14436) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:46 smithi082 bash[33901]: cluster 2023-10-09T04:11:44.856931+0000 mgr.y (mgr.14436) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:48.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:48 smithi153 bash[27868]: cluster 2023-10-09T04:11:46.857567+0000 mgr.y (mgr.14436) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:48 smithi082 bash[27462]: cluster 2023-10-09T04:11:46.857567+0000 mgr.y (mgr.14436) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:48 smithi082 bash[33901]: cluster 2023-10-09T04:11:46.857567+0000 mgr.y (mgr.14436) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:48.911 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:50 smithi153 bash[27868]: cluster 2023-10-09T04:11:48.858664+0000 mgr.y (mgr.14436) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:50 smithi082 bash[27462]: cluster 2023-10-09T04:11:48.858664+0000 mgr.y (mgr.14436) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:50 smithi082 bash[33901]: cluster 2023-10-09T04:11:48.858664+0000 mgr.y (mgr.14436) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:11:51.642 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:11:51.642 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":60,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:11:28.231610+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:11:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:52 smithi082 bash[33901]: cluster 2023-10-09T04:11:50.859130+0000 mgr.y (mgr.14436) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:52 smithi082 bash[33901]: audit 2023-10-09T04:11:51.638139+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.82:0/795562440' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:11:52.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:52 smithi082 bash[27462]: cluster 2023-10-09T04:11:50.859130+0000 mgr.y (mgr.14436) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:52.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:52 smithi082 bash[27462]: audit 2023-10-09T04:11:51.638139+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.82:0/795562440' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:11:52.330 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:11:52.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:11:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:52 smithi153 bash[27868]: cluster 2023-10-09T04:11:50.859130+0000 mgr.y (mgr.14436) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:52 smithi153 bash[27868]: audit 2023-10-09T04:11:51.638139+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.82:0/795562440' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:11:53.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:11:52 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:52] "GET /metrics HTTP/1.1" 200 191359 "" "Prometheus/2.33.4" 2023-10-09T04:11:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:54 smithi153 bash[27868]: cluster 2023-10-09T04:11:52.860144+0000 mgr.y (mgr.14436) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:54.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:11:53 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:11:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:54 smithi082 bash[27462]: cluster 2023-10-09T04:11:52.860144+0000 mgr.y (mgr.14436) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:54 smithi082 bash[33901]: cluster 2023-10-09T04:11:52.860144+0000 mgr.y (mgr.14436) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:56 smithi153 bash[27868]: cluster 2023-10-09T04:11:54.860781+0000 mgr.y (mgr.14436) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:56 smithi082 bash[27462]: cluster 2023-10-09T04:11:54.860781+0000 mgr.y (mgr.14436) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:56.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:56 smithi082 bash[33901]: cluster 2023-10-09T04:11:54.860781+0000 mgr.y (mgr.14436) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:57.108 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:58.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:11:58 smithi082 bash[27462]: cluster 2023-10-09T04:11:56.861474+0000 mgr.y (mgr.14436) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:11:58 smithi082 bash[33901]: cluster 2023-10-09T04:11:56.861474+0000 mgr.y (mgr.14436) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:11:58.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:11:58 smithi153 bash[27868]: cluster 2023-10-09T04:11:56.861474+0000 mgr.y (mgr.14436) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:00.092 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:12:00.092 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":60,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:11:28.231610+0000","last_up_change":"2023-10-09T04:08:35.814376+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6803","nonce":2127587838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6805","nonce":2127587838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6809","nonce":2127587838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":2127587838},{"type":"v1","addr":"172.21.15.82:6807","nonce":2127587838}]},"public_addr":"172.21.15.82:6803/2127587838","cluster_addr":"172.21.15.82:6805/2127587838","heartbeat_back_addr":"172.21.15.82:6809/2127587838","heartbeat_front_addr":"172.21.15.82:6807/2127587838","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6811","nonce":2767022652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6813","nonce":2767022652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6817","nonce":2767022652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2767022652},{"type":"v1","addr":"172.21.15.82:6815","nonce":2767022652}]},"public_addr":"172.21.15.82:6811/2767022652","cluster_addr":"172.21.15.82:6813/2767022652","heartbeat_back_addr":"172.21.15.82:6817/2767022652","heartbeat_front_addr":"172.21.15.82:6815/2767022652","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6819","nonce":3755541550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6821","nonce":3755541550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6825","nonce":3755541550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":3755541550},{"type":"v1","addr":"172.21.15.82:6823","nonce":3755541550}]},"public_addr":"172.21.15.82:6819/3755541550","cluster_addr":"172.21.15.82:6821/3755541550","heartbeat_back_addr":"172.21.15.82:6825/3755541550","heartbeat_front_addr":"172.21.15.82:6823/3755541550","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6827","nonce":3309313210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6829","nonce":3309313210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6833","nonce":3309313210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3309313210},{"type":"v1","addr":"172.21.15.82:6831","nonce":3309313210}]},"public_addr":"172.21.15.82:6827/3309313210","cluster_addr":"172.21.15.82:6829/3309313210","heartbeat_back_addr":"172.21.15.82:6833/3309313210","heartbeat_front_addr":"172.21.15.82:6831/3309313210","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:12:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:00 smithi153 bash[27868]: cluster 2023-10-09T04:11:58.862821+0000 mgr.y (mgr.14436) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:00 smithi153 bash[27868]: audit 2023-10-09T04:12:00.087949+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.82:0/4065263322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:12:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:00 smithi082 bash[27462]: cluster 2023-10-09T04:11:58.862821+0000 mgr.y (mgr.14436) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:00.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:00 smithi082 bash[27462]: audit 2023-10-09T04:12:00.087949+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.82:0/4065263322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:12:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:00 smithi082 bash[33901]: cluster 2023-10-09T04:11:58.862821+0000 mgr.y (mgr.14436) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:00 smithi082 bash[33901]: audit 2023-10-09T04:12:00.087949+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.82:0/4065263322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:12:00.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:12:00.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:12:00.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:12:00.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:12:00.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:12:00.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:12:00.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:12:00.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:12:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:02 smithi153 bash[27868]: cluster 2023-10-09T04:12:00.863356+0000 mgr.y (mgr.14436) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:02.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:02 smithi082 bash[27462]: cluster 2023-10-09T04:12:00.863356+0000 mgr.y (mgr.14436) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:02.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:02 smithi082 bash[33901]: cluster 2023-10-09T04:12:00.863356+0000 mgr.y (mgr.14436) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:03.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:02] "GET /metrics HTTP/1.1" 200 191359 "" "Prometheus/2.33.4" 2023-10-09T04:12:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:04 smithi153 bash[27868]: cluster 2023-10-09T04:12:02.864415+0000 mgr.y (mgr.14436) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:04.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:03 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:04 smithi082 bash[27462]: cluster 2023-10-09T04:12:02.864415+0000 mgr.y (mgr.14436) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:04.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:04 smithi082 bash[33901]: cluster 2023-10-09T04:12:02.864415+0000 mgr.y (mgr.14436) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:05.673 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.678 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.679 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.679 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.679 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.680 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.680 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:05.684 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:06 smithi153 bash[27868]: cluster 2023-10-09T04:12:04.865211+0000 mgr.y (mgr.14436) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:06 smithi082 bash[27462]: cluster 2023-10-09T04:12:04.865211+0000 mgr.y (mgr.14436) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:06 smithi082 bash[33901]: cluster 2023-10-09T04:12:04.865211+0000 mgr.y (mgr.14436) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:07 smithi153 bash[27868]: audit 2023-10-09T04:12:06.856455+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T04:12:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:07 smithi153 bash[27868]: audit 2023-10-09T04:12:06.856793+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T04:12:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:07 smithi153 bash[27868]: audit 2023-10-09T04:12:06.857049+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T04:12:07.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[27462]: audit 2023-10-09T04:12:06.856455+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T04:12:07.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[27462]: audit 2023-10-09T04:12:06.856793+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T04:12:07.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[27462]: audit 2023-10-09T04:12:06.857049+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T04:12:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[33901]: audit 2023-10-09T04:12:06.856455+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-10-09T04:12:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[33901]: audit 2023-10-09T04:12:06.856793+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T04:12:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:07 smithi082 bash[33901]: audit 2023-10-09T04:12:06.857049+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T04:12:08.178 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[33901]: cluster 2023-10-09T04:12:06.865674+0000 mgr.y (mgr.14436) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:08.178 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[33901]: audit 2023-10-09T04:12:07.131180+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T04:12:08.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[33901]: audit 2023-10-09T04:12:07.131263+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T04:12:08.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[33901]: audit 2023-10-09T04:12:07.131327+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T04:12:08.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[33901]: cluster 2023-10-09T04:12:07.131381+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:12:08.179 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[27462]: cluster 2023-10-09T04:12:06.865674+0000 mgr.y (mgr.14436) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:08.179 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[27462]: audit 2023-10-09T04:12:07.131180+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T04:12:08.180 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[27462]: audit 2023-10-09T04:12:07.131263+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T04:12:08.180 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[27462]: audit 2023-10-09T04:12:07.131327+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T04:12:08.180 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:08 smithi082 bash[27462]: cluster 2023-10-09T04:12:07.131381+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:12:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:08 smithi153 bash[27868]: cluster 2023-10-09T04:12:06.865674+0000 mgr.y (mgr.14436) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:08 smithi153 bash[27868]: audit 2023-10-09T04:12:07.131180+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-10-09T04:12:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:08 smithi153 bash[27868]: audit 2023-10-09T04:12:07.131263+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T04:12:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:08 smithi153 bash[27868]: audit 2023-10-09T04:12:07.131327+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T04:12:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:08 smithi153 bash[27868]: cluster 2023-10-09T04:12:07.131381+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:12:09.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:09 smithi153 bash[27868]: cluster 2023-10-09T04:12:08.140803+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:12:09.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:09 smithi082 bash[33901]: cluster 2023-10-09T04:12:08.140803+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:12:09.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:09 smithi082 bash[27462]: cluster 2023-10-09T04:12:08.140803+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:12:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:10 smithi153 bash[27868]: cluster 2023-10-09T04:12:08.866282+0000 mgr.y (mgr.14436) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:10.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:10 smithi082 bash[33901]: cluster 2023-10-09T04:12:08.866282+0000 mgr.y (mgr.14436) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:10.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:10 smithi082 bash[27462]: cluster 2023-10-09T04:12:08.866282+0000 mgr.y (mgr.14436) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:12.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:12 smithi082 bash[33901]: cluster 2023-10-09T04:12:10.866933+0000 mgr.y (mgr.14436) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:12.331 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:12 smithi082 bash[27462]: cluster 2023-10-09T04:12:10.866933+0000 mgr.y (mgr.14436) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:12 smithi153 bash[27868]: cluster 2023-10-09T04:12:10.866933+0000 mgr.y (mgr.14436) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:13.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:12] "GET /metrics HTTP/1.1" 200 191354 "" "Prometheus/2.33.4" 2023-10-09T04:12:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:14 smithi153 bash[27868]: cluster 2023-10-09T04:12:12.867951+0000 mgr.y (mgr.14436) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:14.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:13 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:14 smithi082 bash[33901]: cluster 2023-10-09T04:12:12.867951+0000 mgr.y (mgr.14436) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:14 smithi082 bash[27462]: cluster 2023-10-09T04:12:12.867951+0000 mgr.y (mgr.14436) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:16.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:16 smithi153 bash[27868]: cluster 2023-10-09T04:12:14.868344+0000 mgr.y (mgr.14436) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:16 smithi082 bash[27462]: cluster 2023-10-09T04:12:14.868344+0000 mgr.y (mgr.14436) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:16 smithi082 bash[33901]: cluster 2023-10-09T04:12:14.868344+0000 mgr.y (mgr.14436) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:18 smithi153 bash[27868]: cluster 2023-10-09T04:12:16.869152+0000 mgr.y (mgr.14436) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:18.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:18 smithi082 bash[33901]: cluster 2023-10-09T04:12:16.869152+0000 mgr.y (mgr.14436) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:18.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:18 smithi082 bash[27462]: cluster 2023-10-09T04:12:16.869152+0000 mgr.y (mgr.14436) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:18.551 INFO:teuthology.orchestra.run.smithi082.stdout:206158430254 2023-10-09T04:12:18.551 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:12:20.445 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:20 smithi082 bash[33901]: cluster 2023-10-09T04:12:18.869793+0000 mgr.y (mgr.14436) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:20.445 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:20 smithi082 bash[27462]: cluster 2023-10-09T04:12:18.869793+0000 mgr.y (mgr.14436) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:20 smithi153 bash[27868]: cluster 2023-10-09T04:12:18.869793+0000 mgr.y (mgr.14436) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:20.743 INFO:teuthology.orchestra.run.smithi082.stdout:180388626487 2023-10-09T04:12:20.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:12:20.858 INFO:teuthology.orchestra.run.smithi082.stdout:154618822719 2023-10-09T04:12:20.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:12:21.663 INFO:teuthology.orchestra.run.smithi082.stdout:34359738474 2023-10-09T04:12:21.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:12:22.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:22 smithi082 bash[33901]: cluster 2023-10-09T04:12:20.870223+0000 mgr.y (mgr.14436) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:22.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:22 smithi082 bash[27462]: cluster 2023-10-09T04:12:20.870223+0000 mgr.y (mgr.14436) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:22.511 INFO:teuthology.orchestra.run.smithi082.stdout:55834574946 2023-10-09T04:12:22.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:12:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:22 smithi153 bash[27868]: cluster 2023-10-09T04:12:20.870223+0000 mgr.y (mgr.14436) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:22.725 INFO:teuthology.orchestra.run.smithi082.stdout:77309411418 2023-10-09T04:12:22.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:12:22.897 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:22] "GET /metrics HTTP/1.1" 200 191373 "" "Prometheus/2.33.4" 2023-10-09T04:12:23.415 INFO:teuthology.orchestra.run.smithi082.stdout:128849018952 2023-10-09T04:12:23.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:12:23.486 INFO:teuthology.orchestra.run.smithi082.stdout:107374182479 2023-10-09T04:12:23.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:12:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:24 smithi153 bash[27868]: cluster 2023-10-09T04:12:22.870756+0000 mgr.y (mgr.14436) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:24.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:23 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:24 smithi082 bash[27462]: cluster 2023-10-09T04:12:22.870756+0000 mgr.y (mgr.14436) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:24 smithi082 bash[33901]: cluster 2023-10-09T04:12:22.870756+0000 mgr.y (mgr.14436) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:26.213 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:26.213 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:26.214 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:26.214 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:26.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:26 smithi082 bash[27462]: cluster 2023-10-09T04:12:24.871347+0000 mgr.y (mgr.14436) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:26.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:26 smithi082 bash[33901]: cluster 2023-10-09T04:12:24.871347+0000 mgr.y (mgr.14436) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:26 smithi153 bash[27868]: cluster 2023-10-09T04:12:24.871347+0000 mgr.y (mgr.14436) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:27.228 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:27.229 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:27.229 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:28.237 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:28.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:28 smithi082 bash[27462]: cluster 2023-10-09T04:12:26.872126+0000 mgr.y (mgr.14436) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:28.256 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:28 smithi082 bash[33901]: cluster 2023-10-09T04:12:26.872126+0000 mgr.y (mgr.14436) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:28.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:28 smithi153 bash[27868]: cluster 2023-10-09T04:12:26.872126+0000 mgr.y (mgr.14436) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:30.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:30 smithi082 bash[33901]: cluster 2023-10-09T04:12:28.872860+0000 mgr.y (mgr.14436) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:30.469 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:30 smithi082 bash[27462]: cluster 2023-10-09T04:12:28.872860+0000 mgr.y (mgr.14436) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:30 smithi153 bash[27868]: cluster 2023-10-09T04:12:28.872860+0000 mgr.y (mgr.14436) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:32.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:32 smithi082 bash[33901]: cluster 2023-10-09T04:12:30.873497+0000 mgr.y (mgr.14436) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:32.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:32 smithi082 bash[27462]: cluster 2023-10-09T04:12:30.873497+0000 mgr.y (mgr.14436) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:32 smithi153 bash[27868]: cluster 2023-10-09T04:12:30.873497+0000 mgr.y (mgr.14436) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:33.073 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:32] "GET /metrics HTTP/1.1" 200 191373 "" "Prometheus/2.33.4" 2023-10-09T04:12:34.411 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:34 smithi082 bash[33901]: cluster 2023-10-09T04:12:32.874141+0000 mgr.y (mgr.14436) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:34.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:34 smithi082 bash[27462]: cluster 2023-10-09T04:12:32.874141+0000 mgr.y (mgr.14436) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:34.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:34 smithi153 bash[27868]: cluster 2023-10-09T04:12:32.874141+0000 mgr.y (mgr.14436) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:34.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:34.793 INFO:teuthology.orchestra.run.smithi082.stdout:34359738477 2023-10-09T04:12:34.919 INFO:teuthology.orchestra.run.smithi082.stdout:206158430258 2023-10-09T04:12:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:35 smithi153 bash[27868]: audit 2023-10-09T04:12:34.791801+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.82:0/730616476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:12:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:35 smithi153 bash[27868]: audit 2023-10-09T04:12:34.917608+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.82:0/2404562686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:12:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:35 smithi082 bash[27462]: audit 2023-10-09T04:12:34.791801+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.82:0/730616476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:12:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:35 smithi082 bash[27462]: audit 2023-10-09T04:12:34.917608+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.82:0/2404562686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:12:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:35 smithi082 bash[33901]: audit 2023-10-09T04:12:34.791801+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.82:0/730616476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:12:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:35 smithi082 bash[33901]: audit 2023-10-09T04:12:34.917608+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.82:0/2404562686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:12:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:36 smithi153 bash[27868]: cluster 2023-10-09T04:12:34.874523+0000 mgr.y (mgr.14436) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:36.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:36 smithi082 bash[27462]: cluster 2023-10-09T04:12:34.874523+0000 mgr.y (mgr.14436) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:36.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:36 smithi082 bash[33901]: cluster 2023-10-09T04:12:34.874523+0000 mgr.y (mgr.14436) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:37.351 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738474 got 34359738477 for osd.0 2023-10-09T04:12:37.351 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:38.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:38 smithi082 bash[33901]: cluster 2023-10-09T04:12:36.875156+0000 mgr.y (mgr.14436) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:38 smithi082 bash[27462]: cluster 2023-10-09T04:12:36.875156+0000 mgr.y (mgr.14436) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:38.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:38 smithi153 bash[27868]: cluster 2023-10-09T04:12:36.875156+0000 mgr.y (mgr.14436) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[33901]: audit 2023-10-09T04:12:38.687390+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[33901]: audit 2023-10-09T04:12:38.927063+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[33901]: audit 2023-10-09T04:12:38.952084+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:39.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[27462]: audit 2023-10-09T04:12:38.687390+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:39.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[27462]: audit 2023-10-09T04:12:38.927063+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:39 smithi082 bash[27462]: audit 2023-10-09T04:12:38.952084+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:39.569 INFO:teuthology.orchestra.run.smithi082.stdout:154618822723 2023-10-09T04:12:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:39 smithi153 bash[27868]: audit 2023-10-09T04:12:38.687390+0000 mon.a (mon.0) 715 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:39 smithi153 bash[27868]: audit 2023-10-09T04:12:38.927063+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:39 smithi153 bash[27868]: audit 2023-10-09T04:12:38.952084+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:40 smithi153 bash[27868]: cluster 2023-10-09T04:12:38.875901+0000 mgr.y (mgr.14436) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:40 smithi153 bash[27868]: audit 2023-10-09T04:12:39.228419+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:40 smithi153 bash[27868]: audit 2023-10-09T04:12:39.233286+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:40 smithi153 bash[27868]: audit 2023-10-09T04:12:39.244101+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:40 smithi153 bash[27868]: audit 2023-10-09T04:12:39.567776+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.82:0/2826942266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:12:40.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[27462]: cluster 2023-10-09T04:12:38.875901+0000 mgr.y (mgr.14436) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:40.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[27462]: audit 2023-10-09T04:12:39.228419+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:40.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[27462]: audit 2023-10-09T04:12:39.233286+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[27462]: audit 2023-10-09T04:12:39.244101+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[27462]: audit 2023-10-09T04:12:39.567776+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.82:0/2826942266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[33901]: cluster 2023-10-09T04:12:38.875901+0000 mgr.y (mgr.14436) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[33901]: audit 2023-10-09T04:12:39.228419+0000 mon.a (mon.0) 718 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[33901]: audit 2023-10-09T04:12:39.233286+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[33901]: audit 2023-10-09T04:12:39.244101+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:12:40.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:40 smithi082 bash[33901]: audit 2023-10-09T04:12:39.567776+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.82:0/2826942266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:12:40.679 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430254 got 206158430258 for osd.7 2023-10-09T04:12:40.679 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:41.214 INFO:teuthology.orchestra.run.smithi082.stdout:180388626492 2023-10-09T04:12:41.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:41 smithi082 bash[27462]: audit 2023-10-09T04:12:41.213112+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.82:0/1095255640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:12:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:41 smithi082 bash[33901]: audit 2023-10-09T04:12:41.213112+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.82:0/1095255640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:12:41.494 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822719 got 154618822723 for osd.5 2023-10-09T04:12:41.494 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:41.499 INFO:teuthology.orchestra.run.smithi082.stdout:107374182483 2023-10-09T04:12:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:41 smithi153 bash[27868]: audit 2023-10-09T04:12:41.213112+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.82:0/1095255640' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:12:41.680 INFO:teuthology.orchestra.run.smithi082.stdout:77309411422 2023-10-09T04:12:42.077 INFO:teuthology.orchestra.run.smithi082.stdout:128849018956 2023-10-09T04:12:42.655 INFO:teuthology.orchestra.run.smithi082.stdout:55834574950 2023-10-09T04:12:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:42 smithi153 bash[27868]: cluster 2023-10-09T04:12:40.876484+0000 mgr.y (mgr.14436) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:42 smithi153 bash[27868]: audit 2023-10-09T04:12:41.497893+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.82:0/2337099644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:12:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:42 smithi153 bash[27868]: audit 2023-10-09T04:12:41.678102+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.82:0/1920330328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:12:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:42 smithi153 bash[27868]: audit 2023-10-09T04:12:42.075525+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.82:0/3350036850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[33901]: cluster 2023-10-09T04:12:40.876484+0000 mgr.y (mgr.14436) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[33901]: audit 2023-10-09T04:12:41.497893+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.82:0/2337099644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[33901]: audit 2023-10-09T04:12:41.678102+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.82:0/1920330328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[33901]: audit 2023-10-09T04:12:42.075525+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.82:0/3350036850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[27462]: cluster 2023-10-09T04:12:40.876484+0000 mgr.y (mgr.14436) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:42.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[27462]: audit 2023-10-09T04:12:41.497893+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.82:0/2337099644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:12:42.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[27462]: audit 2023-10-09T04:12:41.678102+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.82:0/1920330328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:12:42.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[27462]: audit 2023-10-09T04:12:42.075525+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.82:0/3350036850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:12:42.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:42 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:42] "GET /metrics HTTP/1.1" 200 191375 "" "Prometheus/2.33.4" 2023-10-09T04:12:43.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:43 smithi153 bash[27868]: audit 2023-10-09T04:12:42.653506+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.82:0/1308577590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:12:43.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:43 smithi082 bash[27462]: audit 2023-10-09T04:12:42.653506+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.82:0/1308577590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:12:43.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:43 smithi082 bash[33901]: audit 2023-10-09T04:12:42.653506+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.82:0/1308577590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:12:43.744 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626487 got 180388626492 for osd.6 2023-10-09T04:12:43.744 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:43.943 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182479 got 107374182483 for osd.3 2023-10-09T04:12:43.943 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:44.267 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:43 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:44.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:44 smithi082 bash[27462]: cluster 2023-10-09T04:12:42.877259+0000 mgr.y (mgr.14436) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:44.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:44 smithi082 bash[33901]: cluster 2023-10-09T04:12:42.877259+0000 mgr.y (mgr.14436) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:44.575 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411418 got 77309411422 for osd.2 2023-10-09T04:12:44.576 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:44 smithi153 bash[27868]: cluster 2023-10-09T04:12:42.877259+0000 mgr.y (mgr.14436) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:45.000 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018952 got 128849018956 for osd.4 2023-10-09T04:12:45.000 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:45.017 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574946 got 55834574950 for osd.1 2023-10-09T04:12:45.017 DEBUG:teuthology.parallel:result is None 2023-10-09T04:12:45.017 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:12:45.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:45 smithi153 bash[27868]: cluster 2023-10-09T04:12:44.877597+0000 mgr.y (mgr.14436) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:45.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:45 smithi082 bash[27462]: cluster 2023-10-09T04:12:44.877597+0000 mgr.y (mgr.14436) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:45 smithi082 bash[33901]: cluster 2023-10-09T04:12:44.877597+0000 mgr.y (mgr.14436) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:48.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:47 smithi082 bash[33901]: cluster 2023-10-09T04:12:46.878164+0000 mgr.y (mgr.14436) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:48.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:47 smithi082 bash[27462]: cluster 2023-10-09T04:12:46.878164+0000 mgr.y (mgr.14436) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:48.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:47 smithi153 bash[27868]: cluster 2023-10-09T04:12:46.878164+0000 mgr.y (mgr.14436) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:49.828 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:50.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:50 smithi082 bash[33901]: cluster 2023-10-09T04:12:48.879216+0000 mgr.y (mgr.14436) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:50.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:50 smithi082 bash[27462]: cluster 2023-10-09T04:12:48.879216+0000 mgr.y (mgr.14436) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:50 smithi153 bash[27868]: cluster 2023-10-09T04:12:48.879216+0000 mgr.y (mgr.14436) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:52 smithi153 bash[27868]: cluster 2023-10-09T04:12:50.879872+0000 mgr.y (mgr.14436) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:52.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:52 smithi082 bash[33901]: cluster 2023-10-09T04:12:50.879872+0000 mgr.y (mgr.14436) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:52.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:52 smithi082 bash[27462]: cluster 2023-10-09T04:12:50.879872+0000 mgr.y (mgr.14436) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:52.644 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:12:52.647 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:12:52.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:12:52 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:52] "GET /metrics HTTP/1.1" 200 191381 "" "Prometheus/2.33.4" 2023-10-09T04:12:53.357 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":126,"stamp":"2023-10-09T04:12:50.879518+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381756,"kb_used_data":4988,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579460,"statfs":{"total":767960285184,"available":765521367040,"internally_reserved":0,"allocated":5107712,"data_stored":2166187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004120"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135042+0000","last_change":"2023-10-09T04:11:27.724509+0000","last_active":"2023-10-09T04:12:07.135042+0000","last_peered":"2023-10-09T04:12:07.135042+0000","last_clean":"2023-10-09T04:12:07.135042+0000","last_became_active":"2023-10-09T04:11:27.724117+0000","last_became_peered":"2023-10-09T04:11:27.724117+0000","last_unstale":"2023-10-09T04:12:07.135042+0000","last_undegraded":"2023-10-09T04:12:07.135042+0000","last_fullsized":"2023-10-09T04:12:07.135042+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131995+0000","last_change":"2023-10-09T04:11:26.209980+0000","last_active":"2023-10-09T04:12:07.131995+0000","last_peered":"2023-10-09T04:12:07.131995+0000","last_clean":"2023-10-09T04:12:07.131995+0000","last_became_active":"2023-10-09T04:11:26.208910+0000","last_became_peered":"2023-10-09T04:11:26.208910+0000","last_unstale":"2023-10-09T04:12:07.131995+0000","last_undegraded":"2023-10-09T04:12:07.131995+0000","last_fullsized":"2023-10-09T04:12:07.131995+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137557+0000","last_change":"2023-10-09T04:11:26.242101+0000","last_active":"2023-10-09T04:12:07.137557+0000","last_peered":"2023-10-09T04:12:07.137557+0000","last_clean":"2023-10-09T04:12:07.137557+0000","last_became_active":"2023-10-09T04:11:26.241769+0000","last_became_peered":"2023-10-09T04:11:26.241769+0000","last_unstale":"2023-10-09T04:12:07.137557+0000","last_undegraded":"2023-10-09T04:12:07.137557+0000","last_fullsized":"2023-10-09T04:12:07.137557+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138010+0000","last_change":"2023-10-09T04:11:26.219164+0000","last_active":"2023-10-09T04:12:07.138010+0000","last_peered":"2023-10-09T04:12:07.138010+0000","last_clean":"2023-10-09T04:12:07.138010+0000","last_became_active":"2023-10-09T04:11:26.218825+0000","last_became_peered":"2023-10-09T04:11:26.218825+0000","last_unstale":"2023-10-09T04:12:07.138010+0000","last_undegraded":"2023-10-09T04:12:07.138010+0000","last_fullsized":"2023-10-09T04:12:07.138010+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135421+0000","last_change":"2023-10-09T04:11:27.726158+0000","last_active":"2023-10-09T04:12:07.135421+0000","last_peered":"2023-10-09T04:12:07.135421+0000","last_clean":"2023-10-09T04:12:07.135421+0000","last_became_active":"2023-10-09T04:11:27.725718+0000","last_became_peered":"2023-10-09T04:11:27.725718+0000","last_unstale":"2023-10-09T04:12:07.135421+0000","last_undegraded":"2023-10-09T04:12:07.135421+0000","last_fullsized":"2023-10-09T04:12:07.135421+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131844+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:12:07.131844+0000","last_peered":"2023-10-09T04:12:07.131844+0000","last_clean":"2023-10-09T04:12:07.131844+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:12:07.131844+0000","last_undegraded":"2023-10-09T04:12:07.131844+0000","last_fullsized":"2023-10-09T04:12:07.131844+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136314+0000","last_change":"2023-10-09T04:11:27.723697+0000","last_active":"2023-10-09T04:12:07.136314+0000","last_peered":"2023-10-09T04:12:07.136314+0000","last_clean":"2023-10-09T04:12:07.136314+0000","last_became_active":"2023-10-09T04:11:27.723096+0000","last_became_peered":"2023-10-09T04:11:27.723096+0000","last_unstale":"2023-10-09T04:12:07.136314+0000","last_undegraded":"2023-10-09T04:12:07.136314+0000","last_fullsized":"2023-10-09T04:12:07.136314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131974+0000","last_change":"2023-10-09T04:11:26.223622+0000","last_active":"2023-10-09T04:12:07.131974+0000","last_peered":"2023-10-09T04:12:07.131974+0000","last_clean":"2023-10-09T04:12:07.131974+0000","last_became_active":"2023-10-09T04:11:26.223360+0000","last_became_peered":"2023-10-09T04:11:26.223360+0000","last_unstale":"2023-10-09T04:12:07.131974+0000","last_undegraded":"2023-10-09T04:12:07.131974+0000","last_fullsized":"2023-10-09T04:12:07.131974+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.294641+0000","last_change":"2023-10-09T04:12:08.294641+0000","last_active":"2023-10-09T04:12:08.294641+0000","last_peered":"2023-10-09T04:12:08.294641+0000","last_clean":"2023-10-09T04:12:08.294641+0000","last_became_active":"2023-10-09T04:12:08.294187+0000","last_became_peered":"2023-10-09T04:12:08.294187+0000","last_unstale":"2023-10-09T04:12:08.294641+0000","last_undegraded":"2023-10-09T04:12:08.294641+0000","last_fullsized":"2023-10-09T04:12:08.294641+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133345+0000","last_change":"2023-10-09T04:11:26.219348+0000","last_active":"2023-10-09T04:12:07.133345+0000","last_peered":"2023-10-09T04:12:07.133345+0000","last_clean":"2023-10-09T04:12:07.133345+0000","last_became_active":"2023-10-09T04:11:26.218973+0000","last_became_peered":"2023-10-09T04:11:26.218973+0000","last_unstale":"2023-10-09T04:12:07.133345+0000","last_undegraded":"2023-10-09T04:12:07.133345+0000","last_fullsized":"2023-10-09T04:12:07.133345+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135346+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:12:07.135346+0000","last_peered":"2023-10-09T04:12:07.135346+0000","last_clean":"2023-10-09T04:12:07.135346+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:12:07.135346+0000","last_undegraded":"2023-10-09T04:12:07.135346+0000","last_fullsized":"2023-10-09T04:12:07.135346+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133361+0000","last_change":"2023-10-09T04:11:26.219648+0000","last_active":"2023-10-09T04:12:07.133361+0000","last_peered":"2023-10-09T04:12:07.133361+0000","last_clean":"2023-10-09T04:12:07.133361+0000","last_became_active":"2023-10-09T04:11:26.219285+0000","last_became_peered":"2023-10-09T04:11:26.219285+0000","last_unstale":"2023-10-09T04:12:07.133361+0000","last_undegraded":"2023-10-09T04:12:07.133361+0000","last_fullsized":"2023-10-09T04:12:07.133361+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190975+0000","last_change":"2023-10-09T04:09:36.312935+0000","last_active":"2023-10-09T04:12:07.190975+0000","last_peered":"2023-10-09T04:12:07.190975+0000","last_clean":"2023-10-09T04:12:07.190975+0000","last_became_active":"2023-10-09T04:09:36.312042+0000","last_became_peered":"2023-10-09T04:09:36.312042+0000","last_unstale":"2023-10-09T04:12:07.190975+0000","last_undegraded":"2023-10-09T04:12:07.190975+0000","last_fullsized":"2023-10-09T04:12:07.190975+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137343+0000","last_change":"2023-10-09T04:11:26.863705+0000","last_active":"2023-10-09T04:12:07.137343+0000","last_peered":"2023-10-09T04:12:07.137343+0000","last_clean":"2023-10-09T04:12:07.137343+0000","last_became_active":"2023-10-09T04:11:26.862898+0000","last_became_peered":"2023-10-09T04:11:26.862898+0000","last_unstale":"2023-10-09T04:12:07.137343+0000","last_undegraded":"2023-10-09T04:12:07.137343+0000","last_fullsized":"2023-10-09T04:12:07.137343+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138459+0000","last_change":"2023-10-09T04:09:37.001556+0000","last_active":"2023-10-09T04:12:07.138459+0000","last_peered":"2023-10-09T04:12:07.138459+0000","last_clean":"2023-10-09T04:12:07.138459+0000","last_became_active":"2023-10-09T04:09:37.000571+0000","last_became_peered":"2023-10-09T04:09:37.000571+0000","last_unstale":"2023-10-09T04:12:07.138459+0000","last_undegraded":"2023-10-09T04:12:07.138459+0000","last_fullsized":"2023-10-09T04:12:07.138459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138407+0000","last_change":"2023-10-09T04:11:26.223038+0000","last_active":"2023-10-09T04:12:07.138407+0000","last_peered":"2023-10-09T04:12:07.138407+0000","last_clean":"2023-10-09T04:12:07.138407+0000","last_became_active":"2023-10-09T04:11:26.222682+0000","last_became_peered":"2023-10-09T04:11:26.222682+0000","last_unstale":"2023-10-09T04:12:07.138407+0000","last_undegraded":"2023-10-09T04:12:07.138407+0000","last_fullsized":"2023-10-09T04:12:07.138407+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133227+0000","last_change":"2023-10-09T04:11:26.862859+0000","last_active":"2023-10-09T04:12:07.133227+0000","last_peered":"2023-10-09T04:12:07.133227+0000","last_clean":"2023-10-09T04:12:07.133227+0000","last_became_active":"2023-10-09T04:11:26.862528+0000","last_became_peered":"2023-10-09T04:11:26.862528+0000","last_unstale":"2023-10-09T04:12:07.133227+0000","last_undegraded":"2023-10-09T04:12:07.133227+0000","last_fullsized":"2023-10-09T04:12:07.133227+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137807+0000","last_change":"2023-10-09T04:11:26.221136+0000","last_active":"2023-10-09T04:12:07.137807+0000","last_peered":"2023-10-09T04:12:07.137807+0000","last_clean":"2023-10-09T04:12:07.137807+0000","last_became_active":"2023-10-09T04:11:26.220650+0000","last_became_peered":"2023-10-09T04:11:26.220650+0000","last_unstale":"2023-10-09T04:12:07.137807+0000","last_undegraded":"2023-10-09T04:12:07.137807+0000","last_fullsized":"2023-10-09T04:12:07.137807+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191961+0000","last_change":"2023-10-09T04:09:37.118061+0000","last_active":"2023-10-09T04:12:07.191961+0000","last_peered":"2023-10-09T04:12:07.191961+0000","last_clean":"2023-10-09T04:12:07.191961+0000","last_became_active":"2023-10-09T04:09:37.117163+0000","last_became_peered":"2023-10-09T04:09:37.117163+0000","last_unstale":"2023-10-09T04:12:07.191961+0000","last_undegraded":"2023-10-09T04:12:07.191961+0000","last_fullsized":"2023-10-09T04:12:07.191961+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133476+0000","last_change":"2023-10-09T04:11:26.239105+0000","last_active":"2023-10-09T04:12:07.133476+0000","last_peered":"2023-10-09T04:12:07.133476+0000","last_clean":"2023-10-09T04:12:07.133476+0000","last_became_active":"2023-10-09T04:11:26.238762+0000","last_became_peered":"2023-10-09T04:11:26.238762+0000","last_unstale":"2023-10-09T04:12:07.133476+0000","last_undegraded":"2023-10-09T04:12:07.133476+0000","last_fullsized":"2023-10-09T04:12:07.133476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131270+0000","last_change":"2023-10-09T04:11:26.219084+0000","last_active":"2023-10-09T04:12:07.131270+0000","last_peered":"2023-10-09T04:12:07.131270+0000","last_clean":"2023-10-09T04:12:07.131270+0000","last_became_active":"2023-10-09T04:11:26.218727+0000","last_became_peered":"2023-10-09T04:11:26.218727+0000","last_unstale":"2023-10-09T04:12:07.131270+0000","last_undegraded":"2023-10-09T04:12:07.131270+0000","last_fullsized":"2023-10-09T04:12:07.131270+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191671+0000","last_change":"2023-10-09T04:09:37.117349+0000","last_active":"2023-10-09T04:12:07.191671+0000","last_peered":"2023-10-09T04:12:07.191671+0000","last_clean":"2023-10-09T04:12:07.191671+0000","last_became_active":"2023-10-09T04:09:37.116674+0000","last_became_peered":"2023-10-09T04:09:37.116674+0000","last_unstale":"2023-10-09T04:12:07.191671+0000","last_undegraded":"2023-10-09T04:12:07.191671+0000","last_fullsized":"2023-10-09T04:12:07.191671+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190792+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:12:07.190792+0000","last_peered":"2023-10-09T04:12:07.190792+0000","last_clean":"2023-10-09T04:12:07.190792+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:12:07.190792+0000","last_undegraded":"2023-10-09T04:12:07.190792+0000","last_fullsized":"2023-10-09T04:12:07.190792+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137799+0000","last_change":"2023-10-09T04:11:26.219926+0000","last_active":"2023-10-09T04:12:07.137799+0000","last_peered":"2023-10-09T04:12:07.137799+0000","last_clean":"2023-10-09T04:12:07.137799+0000","last_became_active":"2023-10-09T04:11:26.219597+0000","last_became_peered":"2023-10-09T04:11:26.219597+0000","last_unstale":"2023-10-09T04:12:07.137799+0000","last_undegraded":"2023-10-09T04:12:07.137799+0000","last_fullsized":"2023-10-09T04:12:07.137799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191474+0000","last_change":"2023-10-09T04:08:38.365338+0000","last_active":"2023-10-09T04:12:07.191474+0000","last_peered":"2023-10-09T04:12:07.191474+0000","last_clean":"2023-10-09T04:12:07.191474+0000","last_became_active":"2023-10-09T04:08:38.364495+0000","last_became_peered":"2023-10-09T04:08:38.364495+0000","last_unstale":"2023-10-09T04:12:07.191474+0000","last_undegraded":"2023-10-09T04:12:07.191474+0000","last_fullsized":"2023-10-09T04:12:07.191474+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131752+0000","last_change":"2023-10-09T04:09:36.312522+0000","last_active":"2023-10-09T04:12:07.131752+0000","last_peered":"2023-10-09T04:12:07.131752+0000","last_clean":"2023-10-09T04:12:07.131752+0000","last_became_active":"2023-10-09T04:09:36.311979+0000","last_became_peered":"2023-10-09T04:09:36.311979+0000","last_unstale":"2023-10-09T04:12:07.131752+0000","last_undegraded":"2023-10-09T04:12:07.131752+0000","last_fullsized":"2023-10-09T04:12:07.131752+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138698+0000","last_change":"2023-10-09T04:11:26.839144+0000","last_active":"2023-10-09T04:12:07.138698+0000","last_peered":"2023-10-09T04:12:07.138698+0000","last_clean":"2023-10-09T04:12:07.138698+0000","last_became_active":"2023-10-09T04:11:26.838499+0000","last_became_peered":"2023-10-09T04:11:26.838499+0000","last_unstale":"2023-10-09T04:12:07.138698+0000","last_undegraded":"2023-10-09T04:12:07.138698+0000","last_fullsized":"2023-10-09T04:12:07.138698+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134937+0000","last_change":"2023-10-09T04:11:26.226361+0000","last_active":"2023-10-09T04:12:07.134937+0000","last_peered":"2023-10-09T04:12:07.134937+0000","last_clean":"2023-10-09T04:12:07.134937+0000","last_became_active":"2023-10-09T04:11:26.225698+0000","last_became_peered":"2023-10-09T04:11:26.225698+0000","last_unstale":"2023-10-09T04:12:07.134937+0000","last_undegraded":"2023-10-09T04:12:07.134937+0000","last_fullsized":"2023-10-09T04:12:07.134937+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134904+0000","last_change":"2023-10-09T04:09:37.118566+0000","last_active":"2023-10-09T04:12:07.134904+0000","last_peered":"2023-10-09T04:12:07.134904+0000","last_clean":"2023-10-09T04:12:07.134904+0000","last_became_active":"2023-10-09T04:09:37.117693+0000","last_became_peered":"2023-10-09T04:09:37.117693+0000","last_unstale":"2023-10-09T04:12:07.134904+0000","last_undegraded":"2023-10-09T04:12:07.134904+0000","last_fullsized":"2023-10-09T04:12:07.134904+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137639+0000","last_change":"2023-10-09T04:11:26.243218+0000","last_active":"2023-10-09T04:12:07.137639+0000","last_peered":"2023-10-09T04:12:07.137639+0000","last_clean":"2023-10-09T04:12:07.137639+0000","last_became_active":"2023-10-09T04:11:26.242904+0000","last_became_peered":"2023-10-09T04:11:26.242904+0000","last_unstale":"2023-10-09T04:12:07.137639+0000","last_undegraded":"2023-10-09T04:12:07.137639+0000","last_fullsized":"2023-10-09T04:12:07.137639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136193+0000","last_change":"2023-10-09T04:11:26.219069+0000","last_active":"2023-10-09T04:12:07.136193+0000","last_peered":"2023-10-09T04:12:07.136193+0000","last_clean":"2023-10-09T04:12:07.136193+0000","last_became_active":"2023-10-09T04:11:26.218722+0000","last_became_peered":"2023-10-09T04:11:26.218722+0000","last_unstale":"2023-10-09T04:12:07.136193+0000","last_undegraded":"2023-10-09T04:12:07.136193+0000","last_fullsized":"2023-10-09T04:12:07.136193+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131555+0000","last_change":"2023-10-09T04:09:36.309784+0000","last_active":"2023-10-09T04:12:07.131555+0000","last_peered":"2023-10-09T04:12:07.131555+0000","last_clean":"2023-10-09T04:12:07.131555+0000","last_became_active":"2023-10-09T04:09:36.309394+0000","last_became_peered":"2023-10-09T04:09:36.309394+0000","last_unstale":"2023-10-09T04:12:07.131555+0000","last_undegraded":"2023-10-09T04:12:07.131555+0000","last_fullsized":"2023-10-09T04:12:07.131555+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134672+0000","last_change":"2023-10-09T04:11:26.838943+0000","last_active":"2023-10-09T04:12:07.134672+0000","last_peered":"2023-10-09T04:12:07.134672+0000","last_clean":"2023-10-09T04:12:07.134672+0000","last_became_active":"2023-10-09T04:11:26.838306+0000","last_became_peered":"2023-10-09T04:11:26.838306+0000","last_unstale":"2023-10-09T04:12:07.134672+0000","last_undegraded":"2023-10-09T04:12:07.134672+0000","last_fullsized":"2023-10-09T04:12:07.134672+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131909+0000","last_change":"2023-10-09T04:11:26.224977+0000","last_active":"2023-10-09T04:12:07.131909+0000","last_peered":"2023-10-09T04:12:07.131909+0000","last_clean":"2023-10-09T04:12:07.131909+0000","last_became_active":"2023-10-09T04:11:26.224643+0000","last_became_peered":"2023-10-09T04:11:26.224643+0000","last_unstale":"2023-10-09T04:12:07.131909+0000","last_undegraded":"2023-10-09T04:12:07.131909+0000","last_fullsized":"2023-10-09T04:12:07.131909+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137480+0000","last_change":"2023-10-09T04:09:37.117854+0000","last_active":"2023-10-09T04:12:07.137480+0000","last_peered":"2023-10-09T04:12:07.137480+0000","last_clean":"2023-10-09T04:12:07.137480+0000","last_became_active":"2023-10-09T04:09:37.117379+0000","last_became_peered":"2023-10-09T04:09:37.117379+0000","last_unstale":"2023-10-09T04:12:07.137480+0000","last_undegraded":"2023-10-09T04:12:07.137480+0000","last_fullsized":"2023-10-09T04:12:07.137480+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138649+0000","last_change":"2023-10-09T04:11:26.242716+0000","last_active":"2023-10-09T04:12:07.138649+0000","last_peered":"2023-10-09T04:12:07.138649+0000","last_clean":"2023-10-09T04:12:07.138649+0000","last_became_active":"2023-10-09T04:11:26.242338+0000","last_became_peered":"2023-10-09T04:11:26.242338+0000","last_unstale":"2023-10-09T04:12:07.138649+0000","last_undegraded":"2023-10-09T04:12:07.138649+0000","last_fullsized":"2023-10-09T04:12:07.138649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137461+0000","last_change":"2023-10-09T04:11:26.222623+0000","last_active":"2023-10-09T04:12:07.137461+0000","last_peered":"2023-10-09T04:12:07.137461+0000","last_clean":"2023-10-09T04:12:07.137461+0000","last_became_active":"2023-10-09T04:11:26.222270+0000","last_became_peered":"2023-10-09T04:11:26.222270+0000","last_unstale":"2023-10-09T04:12:07.137461+0000","last_undegraded":"2023-10-09T04:12:07.137461+0000","last_fullsized":"2023-10-09T04:12:07.137461+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136471+0000","last_change":"2023-10-09T04:11:27.725832+0000","last_active":"2023-10-09T04:12:07.136471+0000","last_peered":"2023-10-09T04:12:07.136471+0000","last_clean":"2023-10-09T04:12:07.136471+0000","last_became_active":"2023-10-09T04:11:27.724932+0000","last_became_peered":"2023-10-09T04:11:27.724932+0000","last_unstale":"2023-10-09T04:12:07.136471+0000","last_undegraded":"2023-10-09T04:12:07.136471+0000","last_fullsized":"2023-10-09T04:12:07.136471+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135313+0000","last_change":"2023-10-09T04:11:26.214762+0000","last_active":"2023-10-09T04:12:07.135313+0000","last_peered":"2023-10-09T04:12:07.135313+0000","last_clean":"2023-10-09T04:12:07.135313+0000","last_became_active":"2023-10-09T04:11:26.214399+0000","last_became_peered":"2023-10-09T04:11:26.214399+0000","last_unstale":"2023-10-09T04:12:07.135313+0000","last_undegraded":"2023-10-09T04:12:07.135313+0000","last_fullsized":"2023-10-09T04:12:07.135313+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135865+0000","last_change":"2023-10-09T04:11:27.725640+0000","last_active":"2023-10-09T04:12:07.135865+0000","last_peered":"2023-10-09T04:12:07.135865+0000","last_clean":"2023-10-09T04:12:07.135865+0000","last_became_active":"2023-10-09T04:11:27.725109+0000","last_became_peered":"2023-10-09T04:11:27.725109+0000","last_unstale":"2023-10-09T04:12:07.135865+0000","last_undegraded":"2023-10-09T04:12:07.135865+0000","last_fullsized":"2023-10-09T04:12:07.135865+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191156+0000","last_change":"2023-10-09T04:11:26.217013+0000","last_active":"2023-10-09T04:12:07.191156+0000","last_peered":"2023-10-09T04:12:07.191156+0000","last_clean":"2023-10-09T04:12:07.191156+0000","last_became_active":"2023-10-09T04:11:26.216695+0000","last_became_peered":"2023-10-09T04:11:26.216695+0000","last_unstale":"2023-10-09T04:12:07.191156+0000","last_undegraded":"2023-10-09T04:12:07.191156+0000","last_fullsized":"2023-10-09T04:12:07.191156+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135772+0000","last_change":"2023-10-09T04:11:27.719835+0000","last_active":"2023-10-09T04:12:07.135772+0000","last_peered":"2023-10-09T04:12:07.135772+0000","last_clean":"2023-10-09T04:12:07.135772+0000","last_became_active":"2023-10-09T04:11:27.719092+0000","last_became_peered":"2023-10-09T04:11:27.719092+0000","last_unstale":"2023-10-09T04:12:07.135772+0000","last_undegraded":"2023-10-09T04:12:07.135772+0000","last_fullsized":"2023-10-09T04:12:07.135772+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138082+0000","last_change":"2023-10-09T04:11:26.225784+0000","last_active":"2023-10-09T04:12:07.138082+0000","last_peered":"2023-10-09T04:12:07.138082+0000","last_clean":"2023-10-09T04:12:07.138082+0000","last_became_active":"2023-10-09T04:11:26.225439+0000","last_became_peered":"2023-10-09T04:11:26.225439+0000","last_unstale":"2023-10-09T04:12:07.138082+0000","last_undegraded":"2023-10-09T04:12:07.138082+0000","last_fullsized":"2023-10-09T04:12:07.138082+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.296192+0000","last_change":"2023-10-09T04:12:08.296192+0000","last_active":"2023-10-09T04:12:08.296192+0000","last_peered":"2023-10-09T04:12:08.296192+0000","last_clean":"2023-10-09T04:12:08.296192+0000","last_became_active":"2023-10-09T04:12:08.295544+0000","last_became_peered":"2023-10-09T04:12:08.295544+0000","last_unstale":"2023-10-09T04:12:08.296192+0000","last_undegraded":"2023-10-09T04:12:08.296192+0000","last_fullsized":"2023-10-09T04:12:08.296192+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131582+0000","last_change":"2023-10-09T04:11:26.224909+0000","last_active":"2023-10-09T04:12:07.131582+0000","last_peered":"2023-10-09T04:12:07.131582+0000","last_clean":"2023-10-09T04:12:07.131582+0000","last_became_active":"2023-10-09T04:11:26.224492+0000","last_became_peered":"2023-10-09T04:11:26.224492+0000","last_unstale":"2023-10-09T04:12:07.131582+0000","last_undegraded":"2023-10-09T04:12:07.131582+0000","last_fullsized":"2023-10-09T04:12:07.131582+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190873+0000","last_change":"2023-10-09T04:11:27.723595+0000","last_active":"2023-10-09T04:12:07.190873+0000","last_peered":"2023-10-09T04:12:07.190873+0000","last_clean":"2023-10-09T04:12:07.190873+0000","last_became_active":"2023-10-09T04:11:27.723172+0000","last_became_peered":"2023-10-09T04:11:27.723172+0000","last_unstale":"2023-10-09T04:12:07.190873+0000","last_undegraded":"2023-10-09T04:12:07.190873+0000","last_fullsized":"2023-10-09T04:12:07.190873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192072+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:12:07.192072+0000","last_peered":"2023-10-09T04:12:07.192072+0000","last_clean":"2023-10-09T04:12:07.192072+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:12:07.192072+0000","last_undegraded":"2023-10-09T04:12:07.192072+0000","last_fullsized":"2023-10-09T04:12:07.192072+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135110+0000","last_change":"2023-10-09T04:11:27.726052+0000","last_active":"2023-10-09T04:12:07.135110+0000","last_peered":"2023-10-09T04:12:07.135110+0000","last_clean":"2023-10-09T04:12:07.135110+0000","last_became_active":"2023-10-09T04:11:27.724179+0000","last_became_peered":"2023-10-09T04:11:27.724179+0000","last_unstale":"2023-10-09T04:12:07.135110+0000","last_undegraded":"2023-10-09T04:12:07.135110+0000","last_fullsized":"2023-10-09T04:12:07.135110+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192146+0000","last_change":"2023-10-09T04:11:26.217513+0000","last_active":"2023-10-09T04:12:07.192146+0000","last_peered":"2023-10-09T04:12:07.192146+0000","last_clean":"2023-10-09T04:12:07.192146+0000","last_became_active":"2023-10-09T04:11:26.217186+0000","last_became_peered":"2023-10-09T04:11:26.217186+0000","last_unstale":"2023-10-09T04:12:07.192146+0000","last_undegraded":"2023-10-09T04:12:07.192146+0000","last_fullsized":"2023-10-09T04:12:07.192146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131421+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:12:07.131421+0000","last_peered":"2023-10-09T04:12:07.131421+0000","last_clean":"2023-10-09T04:12:07.131421+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:12:07.131421+0000","last_undegraded":"2023-10-09T04:12:07.131421+0000","last_fullsized":"2023-10-09T04:12:07.131421+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192386+0000","last_change":"2023-10-09T04:11:26.218190+0000","last_active":"2023-10-09T04:12:07.192386+0000","last_peered":"2023-10-09T04:12:07.192386+0000","last_clean":"2023-10-09T04:12:07.192386+0000","last_became_active":"2023-10-09T04:11:26.217883+0000","last_became_peered":"2023-10-09T04:11:26.217883+0000","last_unstale":"2023-10-09T04:12:07.192386+0000","last_undegraded":"2023-10-09T04:12:07.192386+0000","last_fullsized":"2023-10-09T04:12:07.192386+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138240+0000","last_change":"2023-10-09T04:11:26.864299+0000","last_active":"2023-10-09T04:12:07.138240+0000","last_peered":"2023-10-09T04:12:07.138240+0000","last_clean":"2023-10-09T04:12:07.138240+0000","last_became_active":"2023-10-09T04:11:26.862910+0000","last_became_peered":"2023-10-09T04:11:26.862910+0000","last_unstale":"2023-10-09T04:12:07.138240+0000","last_undegraded":"2023-10-09T04:12:07.138240+0000","last_fullsized":"2023-10-09T04:12:07.138240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190951+0000","last_change":"2023-10-09T04:11:26.216146+0000","last_active":"2023-10-09T04:12:07.190951+0000","last_peered":"2023-10-09T04:12:07.190951+0000","last_clean":"2023-10-09T04:12:07.190951+0000","last_became_active":"2023-10-09T04:11:26.215822+0000","last_became_peered":"2023-10-09T04:11:26.215822+0000","last_unstale":"2023-10-09T04:12:07.190951+0000","last_undegraded":"2023-10-09T04:12:07.190951+0000","last_fullsized":"2023-10-09T04:12:07.190951+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133252+0000","last_change":"2023-10-09T04:11:26.864729+0000","last_active":"2023-10-09T04:12:07.133252+0000","last_peered":"2023-10-09T04:12:07.133252+0000","last_clean":"2023-10-09T04:12:07.133252+0000","last_became_active":"2023-10-09T04:11:26.863731+0000","last_became_peered":"2023-10-09T04:11:26.863731+0000","last_unstale":"2023-10-09T04:12:07.133252+0000","last_undegraded":"2023-10-09T04:12:07.133252+0000","last_fullsized":"2023-10-09T04:12:07.133252+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190704+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:12:07.190704+0000","last_peered":"2023-10-09T04:12:07.190704+0000","last_clean":"2023-10-09T04:12:07.190704+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:12:07.190704+0000","last_undegraded":"2023-10-09T04:12:07.190704+0000","last_fullsized":"2023-10-09T04:12:07.190704+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131629+0000","last_change":"2023-10-09T04:11:27.725820+0000","last_active":"2023-10-09T04:12:07.131629+0000","last_peered":"2023-10-09T04:12:07.131629+0000","last_clean":"2023-10-09T04:12:07.131629+0000","last_became_active":"2023-10-09T04:11:27.723296+0000","last_became_peered":"2023-10-09T04:11:27.723296+0000","last_unstale":"2023-10-09T04:12:07.131629+0000","last_undegraded":"2023-10-09T04:12:07.131629+0000","last_fullsized":"2023-10-09T04:12:07.131629+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131353+0000","last_change":"2023-10-09T04:11:26.223318+0000","last_active":"2023-10-09T04:12:07.131353+0000","last_peered":"2023-10-09T04:12:07.131353+0000","last_clean":"2023-10-09T04:12:07.131353+0000","last_became_active":"2023-10-09T04:11:26.222785+0000","last_became_peered":"2023-10-09T04:11:26.222785+0000","last_unstale":"2023-10-09T04:12:07.131353+0000","last_undegraded":"2023-10-09T04:12:07.131353+0000","last_fullsized":"2023-10-09T04:12:07.131353+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138696+0000","last_change":"2023-10-09T04:11:26.862755+0000","last_active":"2023-10-09T04:12:07.138696+0000","last_peered":"2023-10-09T04:12:07.138696+0000","last_clean":"2023-10-09T04:12:07.138696+0000","last_became_active":"2023-10-09T04:11:26.862377+0000","last_became_peered":"2023-10-09T04:11:26.862377+0000","last_unstale":"2023-10-09T04:12:07.138696+0000","last_undegraded":"2023-10-09T04:12:07.138696+0000","last_fullsized":"2023-10-09T04:12:07.138696+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191221+0000","last_change":"2023-10-09T04:11:26.217595+0000","last_active":"2023-10-09T04:12:07.191221+0000","last_peered":"2023-10-09T04:12:07.191221+0000","last_clean":"2023-10-09T04:12:07.191221+0000","last_became_active":"2023-10-09T04:11:26.217263+0000","last_became_peered":"2023-10-09T04:11:26.217263+0000","last_unstale":"2023-10-09T04:12:07.191221+0000","last_undegraded":"2023-10-09T04:12:07.191221+0000","last_fullsized":"2023-10-09T04:12:07.191221+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136186+0000","last_change":"2023-10-09T04:11:27.724916+0000","last_active":"2023-10-09T04:12:07.136186+0000","last_peered":"2023-10-09T04:12:07.136186+0000","last_clean":"2023-10-09T04:12:07.136186+0000","last_became_active":"2023-10-09T04:11:27.724494+0000","last_became_peered":"2023-10-09T04:11:27.724494+0000","last_unstale":"2023-10-09T04:12:07.136186+0000","last_undegraded":"2023-10-09T04:12:07.136186+0000","last_fullsized":"2023-10-09T04:12:07.136186+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135582+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:12:07.135582+0000","last_peered":"2023-10-09T04:12:07.135582+0000","last_clean":"2023-10-09T04:12:07.135582+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:12:07.135582+0000","last_undegraded":"2023-10-09T04:12:07.135582+0000","last_fullsized":"2023-10-09T04:12:07.135582+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134544+0000","last_change":"2023-10-09T04:11:26.368982+0000","last_active":"2023-10-09T04:12:07.134544+0000","last_peered":"2023-10-09T04:12:07.134544+0000","last_clean":"2023-10-09T04:12:07.134544+0000","last_became_active":"2023-10-09T04:11:26.368463+0000","last_became_peered":"2023-10-09T04:11:26.368463+0000","last_unstale":"2023-10-09T04:12:07.134544+0000","last_undegraded":"2023-10-09T04:12:07.134544+0000","last_fullsized":"2023-10-09T04:12:07.134544+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191861+0000","last_change":"2023-10-09T04:11:26.224975+0000","last_active":"2023-10-09T04:12:07.191861+0000","last_peered":"2023-10-09T04:12:07.191861+0000","last_clean":"2023-10-09T04:12:07.191861+0000","last_became_active":"2023-10-09T04:11:26.224674+0000","last_became_peered":"2023-10-09T04:11:26.224674+0000","last_unstale":"2023-10-09T04:12:07.191861+0000","last_undegraded":"2023-10-09T04:12:07.191861+0000","last_fullsized":"2023-10-09T04:12:07.191861+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137672+0000","last_change":"2023-10-09T04:11:26.863318+0000","last_active":"2023-10-09T04:12:07.137672+0000","last_peered":"2023-10-09T04:12:07.137672+0000","last_clean":"2023-10-09T04:12:07.137672+0000","last_became_active":"2023-10-09T04:11:26.862458+0000","last_became_peered":"2023-10-09T04:11:26.862458+0000","last_unstale":"2023-10-09T04:12:07.137672+0000","last_undegraded":"2023-10-09T04:12:07.137672+0000","last_fullsized":"2023-10-09T04:12:07.137672+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131894+0000","last_change":"2023-10-09T04:11:26.220111+0000","last_active":"2023-10-09T04:12:07.131894+0000","last_peered":"2023-10-09T04:12:07.131894+0000","last_clean":"2023-10-09T04:12:07.131894+0000","last_became_active":"2023-10-09T04:11:26.219798+0000","last_became_peered":"2023-10-09T04:11:26.219798+0000","last_unstale":"2023-10-09T04:12:07.131894+0000","last_undegraded":"2023-10-09T04:12:07.131894+0000","last_fullsized":"2023-10-09T04:12:07.131894+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137558+0000","last_change":"2023-10-09T04:11:26.368923+0000","last_active":"2023-10-09T04:12:07.137558+0000","last_peered":"2023-10-09T04:12:07.137558+0000","last_clean":"2023-10-09T04:12:07.137558+0000","last_became_active":"2023-10-09T04:11:26.368458+0000","last_became_peered":"2023-10-09T04:11:26.368458+0000","last_unstale":"2023-10-09T04:12:07.137558+0000","last_undegraded":"2023-10-09T04:12:07.137558+0000","last_fullsized":"2023-10-09T04:12:07.137558+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131793+0000","last_change":"2023-10-09T04:11:26.216693+0000","last_active":"2023-10-09T04:12:07.131793+0000","last_peered":"2023-10-09T04:12:07.131793+0000","last_clean":"2023-10-09T04:12:07.131793+0000","last_became_active":"2023-10-09T04:11:26.216375+0000","last_became_peered":"2023-10-09T04:11:26.216375+0000","last_unstale":"2023-10-09T04:12:07.131793+0000","last_undegraded":"2023-10-09T04:12:07.131793+0000","last_fullsized":"2023-10-09T04:12:07.131793+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191005+0000","last_change":"2023-10-09T04:11:27.721427+0000","last_active":"2023-10-09T04:12:07.191005+0000","last_peered":"2023-10-09T04:12:07.191005+0000","last_clean":"2023-10-09T04:12:07.191005+0000","last_became_active":"2023-10-09T04:11:27.720974+0000","last_became_peered":"2023-10-09T04:11:27.720974+0000","last_unstale":"2023-10-09T04:12:07.191005+0000","last_undegraded":"2023-10-09T04:12:07.191005+0000","last_fullsized":"2023-10-09T04:12:07.191005+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137973+0000","last_change":"2023-10-09T04:11:26.221810+0000","last_active":"2023-10-09T04:12:07.137973+0000","last_peered":"2023-10-09T04:12:07.137973+0000","last_clean":"2023-10-09T04:12:07.137973+0000","last_became_active":"2023-10-09T04:11:26.221462+0000","last_became_peered":"2023-10-09T04:11:26.221462+0000","last_unstale":"2023-10-09T04:12:07.137973+0000","last_undegraded":"2023-10-09T04:12:07.137973+0000","last_fullsized":"2023-10-09T04:12:07.137973+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133293+0000","last_change":"2023-10-09T04:11:26.866153+0000","last_active":"2023-10-09T04:12:07.133293+0000","last_peered":"2023-10-09T04:12:07.133293+0000","last_clean":"2023-10-09T04:12:07.133293+0000","last_became_active":"2023-10-09T04:11:26.865701+0000","last_became_peered":"2023-10-09T04:11:26.865701+0000","last_unstale":"2023-10-09T04:12:07.133293+0000","last_undegraded":"2023-10-09T04:12:07.133293+0000","last_fullsized":"2023-10-09T04:12:07.133293+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134764+0000","last_change":"2023-10-09T04:11:26.227226+0000","last_active":"2023-10-09T04:12:07.134764+0000","last_peered":"2023-10-09T04:12:07.134764+0000","last_clean":"2023-10-09T04:12:07.134764+0000","last_became_active":"2023-10-09T04:11:26.226823+0000","last_became_peered":"2023-10-09T04:11:26.226823+0000","last_unstale":"2023-10-09T04:12:07.134764+0000","last_undegraded":"2023-10-09T04:12:07.134764+0000","last_fullsized":"2023-10-09T04:12:07.134764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.295233+0000","last_change":"2023-10-09T04:12:08.295233+0000","last_active":"2023-10-09T04:12:08.295233+0000","last_peered":"2023-10-09T04:12:08.295233+0000","last_clean":"2023-10-09T04:12:08.295233+0000","last_became_active":"2023-10-09T04:12:08.294781+0000","last_became_peered":"2023-10-09T04:12:08.294781+0000","last_unstale":"2023-10-09T04:12:08.295233+0000","last_undegraded":"2023-10-09T04:12:08.295233+0000","last_fullsized":"2023-10-09T04:12:08.295233+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135167+0000","last_change":"2023-10-09T04:11:26.222061+0000","last_active":"2023-10-09T04:12:07.135167+0000","last_peered":"2023-10-09T04:12:07.135167+0000","last_clean":"2023-10-09T04:12:07.135167+0000","last_became_active":"2023-10-09T04:11:26.221709+0000","last_became_peered":"2023-10-09T04:11:26.221709+0000","last_unstale":"2023-10-09T04:12:07.135167+0000","last_undegraded":"2023-10-09T04:12:07.135167+0000","last_fullsized":"2023-10-09T04:12:07.135167+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430261,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89000000000000001,"15min":0.88500000000000001},"min":{"1min":0.61599999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.218,"5min":1.25,"15min":1.25},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90400000000000003,"15min":0.91300000000000003},"min":{"1min":0.62,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2529999999999999,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.96599999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.872,"15min":0.84599999999999997},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.2270000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.95799999999999996,"15min":0.94399999999999995},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4690000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1.038}]},{"osd":2,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.86499999999999999,"15min":0.88300000000000001},"min":{"1min":0.64100000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.097,"5min":1.425,"15min":1.425},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.92500000000000004,"15min":0.90500000000000003},"min":{"1min":0.58199999999999996,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2989999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.1439999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.90900000000000003,"15min":0.89200000000000002},"min":{"1min":0.74299999999999999,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.167,"5min":1.206,"15min":1.206},"last":1.167},{"interface":"front","average":{"1min":0.997,"5min":0.96499999999999997,"15min":0.94699999999999995},"min":{"1min":0.72799999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2470000000000001,"5min":2.165,"15min":2.165},"last":0.72799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.89400000000000002,"15min":0.87},"min":{"1min":0.66200000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.91400000000000003,"15min":0.86199999999999999},"min":{"1min":0.748,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.296,"5min":1.349,"15min":1.349},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.95199999999999996,"15min":0.90900000000000003},"min":{"1min":0.69799999999999995,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.3109999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.995,"15min":0.95699999999999996},"min":{"1min":0.69099999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5269999999999999,"5min":1.673,"15min":1.673},"last":0.82799999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.0289999999999999,"15min":0.98399999999999999},"min":{"1min":0.84899999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.569,"5min":1.591,"15min":1.591},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0800000000000001,"15min":1.1160000000000001},"min":{"1min":0.89100000000000001,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4510000000000001,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":1.256}]}]},{"osd":6,"up_from":42,"seq":180388626494,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":271554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.84599999999999997,"15min":0.83499999999999996},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.1850000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.57699999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.25,"5min":1.296,"15min":1.296},"last":1.0920000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.91100000000000003,"15min":0.93500000000000005},"min":{"1min":0.627,"5min":0.502,"15min":0.502},"max":{"1min":1.1739999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.04},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.93700000000000006,"15min":0.95799999999999996},"min":{"1min":0.77700000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.171,"5min":1.228,"15min":1.228},"last":0.94899999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.90300000000000002,"15min":0.88400000000000001},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.194,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.89600000000000002,"15min":0.91200000000000003},"min":{"1min":0.54100000000000004,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1319999999999999,"5min":1.242,"15min":1.242},"last":1.125}]},{"osd":3,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.85799999999999998,"15min":0.875},"min":{"1min":0.68999999999999995,"5min":0.496,"15min":0.496},"max":{"1min":1.2509999999999999,"5min":1.278,"15min":1.278},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.96299999999999997,"15min":0.96099999999999997},"min":{"1min":0.63,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.254,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.78900000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.83499999999999996,"15min":0.79000000000000004},"min":{"1min":0.68700000000000006,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1719999999999999,"5min":1.329,"15min":1.329},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.86799999999999999,"15min":0.85599999999999998},"min":{"1min":0.66600000000000004,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2370000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.012}]},{"osd":5,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.94299999999999995,"15min":0.88600000000000001},"min":{"1min":0.78700000000000003,"5min":0.434,"15min":0.434},"max":{"1min":1.306,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.036,"5min":0.94099999999999995,"15min":0.92500000000000004},"min":{"1min":0.79700000000000004,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.224,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.083}]},{"osd":7,"last update":"Mon Oct 9 04:12:46 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0329999999999999,"15min":0.98499999999999999},"min":{"1min":0.86699999999999999,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.393,"5min":1.427,"15min":1.427},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.1060000000000001,"15min":1.056},"min":{"1min":0.94299999999999995,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.4870000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.2350000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574952,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67000000000000004,"15min":0.58099999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.84899999999999998,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.68999999999999995,"15min":0.60099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.97399999999999998,"5min":1.149,"15min":1.149},"last":0.93899999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.72299999999999998,"15min":0.65300000000000002},"min":{"1min":0.38100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.048,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.179},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.72799999999999998,"15min":0.66500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.1799999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.85399999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.81699999999999995,"15min":0.80100000000000005},"min":{"1min":0.51300000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0649999999999999,"5min":1.151,"15min":1.151},"last":1.006},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.85699999999999998,"15min":0.81799999999999995},"min":{"1min":0.46600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.218,"5min":1.258,"15min":1.258},"last":1.069}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.91300000000000003,"15min":0.92100000000000004},"min":{"1min":0.65900000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.254,"5min":1.347,"15min":1.347},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.0069999999999999,"15min":0.98699999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.391,"5min":1.641,"15min":1.641},"last":1.2809999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.98099999999999998,"15min":1.0089999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.353,"5min":1.421,"15min":1.421},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.97199999999999998,"15min":1.012},"min":{"1min":0.72199999999999998,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2669999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.099}]},{"osd":6,"last update":"Mon Oct 9 04:12:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.032,"15min":1.0349999999999999},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3080000000000001,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.151},{"interface":"front","average":{"1min":1.048,"5min":1.032,"15min":1.0329999999999999},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2370000000000001,"5min":3.327,"15min":3.327},"last":1.2470000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.0109999999999999,"15min":1.0329999999999999},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2889999999999999,"5min":1.502,"15min":1.502},"last":1.323},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0760000000000001,"15min":1.1100000000000001},"min":{"1min":0.69999999999999996,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.464,"5min":1.706,"15min":1.706},"last":1.347}]}]},{"osd":0,"up_from":8,"seq":34359738480,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.65000000000000002,"15min":0.59199999999999997},"min":{"1min":0.38200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.873,"5min":1.012,"15min":1.012},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.69999999999999996,"15min":0.626},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.94499999999999995,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.1100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.71699999999999997,"15min":0.66200000000000003},"min":{"1min":0.42799999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.86799999999999999,"5min":1.444,"15min":1.444},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.73699999999999999,"15min":0.67700000000000005},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.0860000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.95999999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.73299999999999998,"15min":0.73499999999999999},"min":{"1min":0.44,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.1240000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80600000000000005},"min":{"1min":0.49299999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.2270000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.0620000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.98199999999999998,"15min":0.97499999999999998},"min":{"1min":0.65500000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.506,"15min":1.506},"last":1.131},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.98899999999999999,"15min":0.96299999999999997},"min":{"1min":0.56100000000000005,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.0960000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.042,"15min":1.0449999999999999},"min":{"1min":0.61799999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.4690000000000001,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":1,"5min":1.0169999999999999,"15min":1.012},"min":{"1min":0.71199999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":1.417}]},{"osd":6,"last update":"Mon Oct 9 04:12:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0589999999999999,"15min":1.087},"min":{"1min":0.73799999999999999,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.464},{"interface":"front","average":{"1min":1.089,"5min":1.123,"15min":1.163},"min":{"1min":0.79200000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":1.3680000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.071,"15min":1.089},"min":{"1min":0.71999999999999997,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.675,"5min":1.675,"15min":1.675},"last":1.53},{"interface":"front","average":{"1min":1.077,"5min":1.129,"15min":1.149},"min":{"1min":0.77300000000000002,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.4199999999999999,"5min":1.448,"15min":1.448},"last":1.444}]}]},{"osd":2,"up_from":18,"seq":77309411424,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.65800000000000003,"15min":0.60499999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.745,"5min":0.71499999999999997,"15min":0.64800000000000002},"min":{"1min":0.40600000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.85199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.72599999999999998,"15min":0.66400000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.73699999999999999,"15min":0.68300000000000005},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.92600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:12:03 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.74299999999999999,"15min":0.76700000000000002},"min":{"1min":0.57299999999999995,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.1240000000000001,"5min":1.137,"15min":1.137},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.79000000000000004,"15min":0.80100000000000005},"min":{"1min":0.47499999999999998,"5min":0.435,"15min":0.435},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.81899999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.95699999999999996,"15min":0.93500000000000005},"min":{"1min":0.58099999999999996,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2809999999999999,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.02,"5min":0.997,"15min":0.95999999999999996},"min":{"1min":0.70999999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.423,"5min":1.468,"15min":1.468},"last":0.98099999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.008},"min":{"1min":0.71099999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.101},{"interface":"front","average":{"1min":1.04,"5min":1.02,"15min":1.0209999999999999},"min":{"1min":0.75600000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.95399999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0620000000000001,"15min":1.044},"min":{"1min":0.86199999999999999,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.663,"5min":1.663,"15min":1.663},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.0840000000000001,"15min":1.093},"min":{"1min":0.93899999999999995,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.736,"5min":1.736,"15min":1.736},"last":1.2150000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0900000000000001,"15min":1.1100000000000001},"min":{"1min":0.63,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.385,"5min":1.619,"15min":1.619},"last":1.153},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.087,"15min":1.069},"min":{"1min":0.76500000000000001,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.401,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":1.1910000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182485,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.69999999999999996},"min":{"1min":0.44600000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.81899999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.69699999999999995,"15min":0.70599999999999996},"min":{"1min":0.37,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.95999999999999996,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.77400000000000002,"15min":0.77600000000000002},"min":{"1min":0.47099999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.1160000000000001,"5min":1.268,"15min":1.4910000000000001},"last":0.63500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.432,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.218,"5min":1.597,"15min":1.597},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.95599999999999996,"5min":1.522,"15min":1.522},"last":0.90300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.92200000000000004,"15min":0.90000000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.173,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0189999999999999,"15min":1.0229999999999999},"min":{"1min":0.67400000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.2969999999999999,"5min":1.482,"15min":1.482},"last":1.0389999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.97899999999999998,"15min":1.0169999999999999},"min":{"1min":0.63700000000000001,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.4039999999999999,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.876},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0680000000000001,"15min":1.0860000000000001},"min":{"1min":0.84999999999999998,"5min":0.624,"15min":0.624},"max":{"1min":1.351,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.94799999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:12:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0569999999999999,"15min":1.0609999999999999},"min":{"1min":0.59999999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3799999999999999,"5min":1.583,"15min":1.583},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.119,"15min":1.127},"min":{"1min":0.82499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.4159999999999999,"5min":1.552,"15min":1.552},"last":1.0880000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0509999999999999,"15min":1.1000000000000001},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.2669999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0840000000000001,"15min":1.1060000000000001},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.454,"5min":1.621,"15min":1.621},"last":1.075}]}]},{"osd":4,"up_from":30,"seq":128849018958,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.85799999999999998,"15min":0.83599999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.159,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91200000000000003,"15min":0.86399999999999999},"min":{"1min":0.59799999999999998,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.1299999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":1.028}]},{"osd":1,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84999999999999998,"15min":0.83899999999999997},"min":{"1min":0.58599999999999997,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.137,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.88300000000000001,"15min":0.83699999999999997},"min":{"1min":0.63900000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.087,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.73699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.89600000000000002,"15min":0.86499999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1140000000000001,"5min":1.409,"15min":1.409},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.93400000000000005,"15min":0.89400000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.169,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.1000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93899999999999995,"15min":0.88100000000000001},"min":{"1min":0.71999999999999997,"5min":0.621,"15min":0.621},"max":{"1min":1.206,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.88400000000000001,"15min":0.88900000000000001},"min":{"1min":0.68300000000000005,"5min":0.60199999999999998,"15min":0.58099999999999996},"max":{"1min":1.044,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.82099999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.84799999999999998,"15min":0.77800000000000002},"min":{"1min":0.52500000000000002,"5min":0.433,"15min":0.433},"max":{"1min":1.284,"5min":1.579,"15min":1.579},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.82799999999999996,"15min":0.73099999999999998},"min":{"1min":0.60899999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.147,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.88,"15min":0.879},"min":{"1min":0.65800000000000003,"5min":0.496,"15min":0.496},"max":{"1min":1.157,"5min":1.609,"15min":1.609},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.02,"5min":0.95899999999999996,"15min":0.93700000000000006},"min":{"1min":0.48999999999999999,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.5009999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.872}]},{"osd":7,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.099,"15min":1.119},"min":{"1min":0.65900000000000003,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.353,"5min":1.653,"15min":1.653},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.054,"5min":1.042,"15min":1.056},"min":{"1min":0.73999999999999999,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2310000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":1.125}]}]},{"osd":5,"up_from":36,"seq":154618822725,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.85899999999999999,"15min":0.84099999999999997},"min":{"1min":0.65700000000000003,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.246,"5min":1.246,"15min":1.246},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86699999999999999,"15min":0.84499999999999997},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.153,"5min":1.232,"15min":1.232},"last":1.304}]},{"osd":1,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.64400000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.093,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.879,"15min":0.85799999999999998},"min":{"1min":0.55600000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":1.125}]},{"osd":2,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.88800000000000001,"15min":0.88},"min":{"1min":0.72299999999999998,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.361,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.89400000000000002,"15min":0.88200000000000001},"min":{"1min":0.56999999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.85199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.90600000000000003,"15min":0.89000000000000001},"min":{"1min":0.53900000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.177},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.95499999999999996,"15min":0.92300000000000004},"min":{"1min":0.74199999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2310000000000001,"5min":1.458,"15min":1.458},"last":1.1970000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.86799999999999999,"15min":0.79200000000000004},"min":{"1min":0.52300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2869999999999999,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.92800000000000005,"15min":0.877},"min":{"1min":0.49299999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":1.258}]},{"osd":6,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.876,"15min":0.77800000000000002},"min":{"1min":0.47799999999999998,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":1.4179999999999999},{"interface":"front","average":{"1min":1.131,"5min":0.93899999999999995,"15min":0.88100000000000001},"min":{"1min":0.60899999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":1.3300000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.0209999999999999,"15min":0.92100000000000004},"min":{"1min":0.68899999999999995,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.4490000000000001,"5min":1.486,"15min":1.486},"last":1.282},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.056,"15min":0.96399999999999997},"min":{"1min":0.627,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.357,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.4650000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:53.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:54 smithi153 bash[27868]: audit 2023-10-09T04:12:52.625881+0000 mgr.y (mgr.14436) 146 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:54 smithi153 bash[27868]: cluster 2023-10-09T04:12:52.880927+0000 mgr.y (mgr.14436) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:54.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:12:53 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:12:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:54 smithi082 bash[27462]: audit 2023-10-09T04:12:52.625881+0000 mgr.y (mgr.14436) 146 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:54.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:54 smithi082 bash[27462]: cluster 2023-10-09T04:12:52.880927+0000 mgr.y (mgr.14436) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:54 smithi082 bash[33901]: audit 2023-10-09T04:12:52.625881+0000 mgr.y (mgr.14436) 146 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:54 smithi082 bash[33901]: cluster 2023-10-09T04:12:52.880927+0000 mgr.y (mgr.14436) 147 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:56 smithi153 bash[27868]: cluster 2023-10-09T04:12:54.881458+0000 mgr.y (mgr.14436) 148 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:56 smithi082 bash[27462]: cluster 2023-10-09T04:12:54.881458+0000 mgr.y (mgr.14436) 148 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:56.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:56 smithi082 bash[33901]: cluster 2023-10-09T04:12:54.881458+0000 mgr.y (mgr.14436) 148 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:58.175 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:58.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:12:58 smithi153 bash[27868]: cluster 2023-10-09T04:12:56.882107+0000 mgr.y (mgr.14436) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:12:58 smithi082 bash[27462]: cluster 2023-10-09T04:12:56.882107+0000 mgr.y (mgr.14436) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:12:58.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:12:58 smithi082 bash[33901]: cluster 2023-10-09T04:12:56.882107+0000 mgr.y (mgr.14436) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:00.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:00 smithi082 bash[27462]: cluster 2023-10-09T04:12:58.882751+0000 mgr.y (mgr.14436) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:00 smithi082 bash[33901]: cluster 2023-10-09T04:12:58.882751+0000 mgr.y (mgr.14436) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:00 smithi153 bash[27868]: cluster 2023-10-09T04:12:58.882751+0000 mgr.y (mgr.14436) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:00.895 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:13:00.897 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:13:01.659 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":130,"stamp":"2023-10-09T04:12:58.882356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381756,"kb_used_data":4988,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579460,"statfs":{"total":767960285184,"available":765521367040,"internally_reserved":0,"allocated":5107712,"data_stored":2166187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004504"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135042+0000","last_change":"2023-10-09T04:11:27.724509+0000","last_active":"2023-10-09T04:12:07.135042+0000","last_peered":"2023-10-09T04:12:07.135042+0000","last_clean":"2023-10-09T04:12:07.135042+0000","last_became_active":"2023-10-09T04:11:27.724117+0000","last_became_peered":"2023-10-09T04:11:27.724117+0000","last_unstale":"2023-10-09T04:12:07.135042+0000","last_undegraded":"2023-10-09T04:12:07.135042+0000","last_fullsized":"2023-10-09T04:12:07.135042+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131995+0000","last_change":"2023-10-09T04:11:26.209980+0000","last_active":"2023-10-09T04:12:07.131995+0000","last_peered":"2023-10-09T04:12:07.131995+0000","last_clean":"2023-10-09T04:12:07.131995+0000","last_became_active":"2023-10-09T04:11:26.208910+0000","last_became_peered":"2023-10-09T04:11:26.208910+0000","last_unstale":"2023-10-09T04:12:07.131995+0000","last_undegraded":"2023-10-09T04:12:07.131995+0000","last_fullsized":"2023-10-09T04:12:07.131995+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137557+0000","last_change":"2023-10-09T04:11:26.242101+0000","last_active":"2023-10-09T04:12:07.137557+0000","last_peered":"2023-10-09T04:12:07.137557+0000","last_clean":"2023-10-09T04:12:07.137557+0000","last_became_active":"2023-10-09T04:11:26.241769+0000","last_became_peered":"2023-10-09T04:11:26.241769+0000","last_unstale":"2023-10-09T04:12:07.137557+0000","last_undegraded":"2023-10-09T04:12:07.137557+0000","last_fullsized":"2023-10-09T04:12:07.137557+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138010+0000","last_change":"2023-10-09T04:11:26.219164+0000","last_active":"2023-10-09T04:12:07.138010+0000","last_peered":"2023-10-09T04:12:07.138010+0000","last_clean":"2023-10-09T04:12:07.138010+0000","last_became_active":"2023-10-09T04:11:26.218825+0000","last_became_peered":"2023-10-09T04:11:26.218825+0000","last_unstale":"2023-10-09T04:12:07.138010+0000","last_undegraded":"2023-10-09T04:12:07.138010+0000","last_fullsized":"2023-10-09T04:12:07.138010+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135421+0000","last_change":"2023-10-09T04:11:27.726158+0000","last_active":"2023-10-09T04:12:07.135421+0000","last_peered":"2023-10-09T04:12:07.135421+0000","last_clean":"2023-10-09T04:12:07.135421+0000","last_became_active":"2023-10-09T04:11:27.725718+0000","last_became_peered":"2023-10-09T04:11:27.725718+0000","last_unstale":"2023-10-09T04:12:07.135421+0000","last_undegraded":"2023-10-09T04:12:07.135421+0000","last_fullsized":"2023-10-09T04:12:07.135421+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131844+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:12:07.131844+0000","last_peered":"2023-10-09T04:12:07.131844+0000","last_clean":"2023-10-09T04:12:07.131844+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:12:07.131844+0000","last_undegraded":"2023-10-09T04:12:07.131844+0000","last_fullsized":"2023-10-09T04:12:07.131844+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136314+0000","last_change":"2023-10-09T04:11:27.723697+0000","last_active":"2023-10-09T04:12:07.136314+0000","last_peered":"2023-10-09T04:12:07.136314+0000","last_clean":"2023-10-09T04:12:07.136314+0000","last_became_active":"2023-10-09T04:11:27.723096+0000","last_became_peered":"2023-10-09T04:11:27.723096+0000","last_unstale":"2023-10-09T04:12:07.136314+0000","last_undegraded":"2023-10-09T04:12:07.136314+0000","last_fullsized":"2023-10-09T04:12:07.136314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131974+0000","last_change":"2023-10-09T04:11:26.223622+0000","last_active":"2023-10-09T04:12:07.131974+0000","last_peered":"2023-10-09T04:12:07.131974+0000","last_clean":"2023-10-09T04:12:07.131974+0000","last_became_active":"2023-10-09T04:11:26.223360+0000","last_became_peered":"2023-10-09T04:11:26.223360+0000","last_unstale":"2023-10-09T04:12:07.131974+0000","last_undegraded":"2023-10-09T04:12:07.131974+0000","last_fullsized":"2023-10-09T04:12:07.131974+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.294641+0000","last_change":"2023-10-09T04:12:08.294641+0000","last_active":"2023-10-09T04:12:08.294641+0000","last_peered":"2023-10-09T04:12:08.294641+0000","last_clean":"2023-10-09T04:12:08.294641+0000","last_became_active":"2023-10-09T04:12:08.294187+0000","last_became_peered":"2023-10-09T04:12:08.294187+0000","last_unstale":"2023-10-09T04:12:08.294641+0000","last_undegraded":"2023-10-09T04:12:08.294641+0000","last_fullsized":"2023-10-09T04:12:08.294641+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133345+0000","last_change":"2023-10-09T04:11:26.219348+0000","last_active":"2023-10-09T04:12:07.133345+0000","last_peered":"2023-10-09T04:12:07.133345+0000","last_clean":"2023-10-09T04:12:07.133345+0000","last_became_active":"2023-10-09T04:11:26.218973+0000","last_became_peered":"2023-10-09T04:11:26.218973+0000","last_unstale":"2023-10-09T04:12:07.133345+0000","last_undegraded":"2023-10-09T04:12:07.133345+0000","last_fullsized":"2023-10-09T04:12:07.133345+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135346+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:12:07.135346+0000","last_peered":"2023-10-09T04:12:07.135346+0000","last_clean":"2023-10-09T04:12:07.135346+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:12:07.135346+0000","last_undegraded":"2023-10-09T04:12:07.135346+0000","last_fullsized":"2023-10-09T04:12:07.135346+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133361+0000","last_change":"2023-10-09T04:11:26.219648+0000","last_active":"2023-10-09T04:12:07.133361+0000","last_peered":"2023-10-09T04:12:07.133361+0000","last_clean":"2023-10-09T04:12:07.133361+0000","last_became_active":"2023-10-09T04:11:26.219285+0000","last_became_peered":"2023-10-09T04:11:26.219285+0000","last_unstale":"2023-10-09T04:12:07.133361+0000","last_undegraded":"2023-10-09T04:12:07.133361+0000","last_fullsized":"2023-10-09T04:12:07.133361+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190975+0000","last_change":"2023-10-09T04:09:36.312935+0000","last_active":"2023-10-09T04:12:07.190975+0000","last_peered":"2023-10-09T04:12:07.190975+0000","last_clean":"2023-10-09T04:12:07.190975+0000","last_became_active":"2023-10-09T04:09:36.312042+0000","last_became_peered":"2023-10-09T04:09:36.312042+0000","last_unstale":"2023-10-09T04:12:07.190975+0000","last_undegraded":"2023-10-09T04:12:07.190975+0000","last_fullsized":"2023-10-09T04:12:07.190975+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137343+0000","last_change":"2023-10-09T04:11:26.863705+0000","last_active":"2023-10-09T04:12:07.137343+0000","last_peered":"2023-10-09T04:12:07.137343+0000","last_clean":"2023-10-09T04:12:07.137343+0000","last_became_active":"2023-10-09T04:11:26.862898+0000","last_became_peered":"2023-10-09T04:11:26.862898+0000","last_unstale":"2023-10-09T04:12:07.137343+0000","last_undegraded":"2023-10-09T04:12:07.137343+0000","last_fullsized":"2023-10-09T04:12:07.137343+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138459+0000","last_change":"2023-10-09T04:09:37.001556+0000","last_active":"2023-10-09T04:12:07.138459+0000","last_peered":"2023-10-09T04:12:07.138459+0000","last_clean":"2023-10-09T04:12:07.138459+0000","last_became_active":"2023-10-09T04:09:37.000571+0000","last_became_peered":"2023-10-09T04:09:37.000571+0000","last_unstale":"2023-10-09T04:12:07.138459+0000","last_undegraded":"2023-10-09T04:12:07.138459+0000","last_fullsized":"2023-10-09T04:12:07.138459+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138407+0000","last_change":"2023-10-09T04:11:26.223038+0000","last_active":"2023-10-09T04:12:07.138407+0000","last_peered":"2023-10-09T04:12:07.138407+0000","last_clean":"2023-10-09T04:12:07.138407+0000","last_became_active":"2023-10-09T04:11:26.222682+0000","last_became_peered":"2023-10-09T04:11:26.222682+0000","last_unstale":"2023-10-09T04:12:07.138407+0000","last_undegraded":"2023-10-09T04:12:07.138407+0000","last_fullsized":"2023-10-09T04:12:07.138407+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133227+0000","last_change":"2023-10-09T04:11:26.862859+0000","last_active":"2023-10-09T04:12:07.133227+0000","last_peered":"2023-10-09T04:12:07.133227+0000","last_clean":"2023-10-09T04:12:07.133227+0000","last_became_active":"2023-10-09T04:11:26.862528+0000","last_became_peered":"2023-10-09T04:11:26.862528+0000","last_unstale":"2023-10-09T04:12:07.133227+0000","last_undegraded":"2023-10-09T04:12:07.133227+0000","last_fullsized":"2023-10-09T04:12:07.133227+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137807+0000","last_change":"2023-10-09T04:11:26.221136+0000","last_active":"2023-10-09T04:12:07.137807+0000","last_peered":"2023-10-09T04:12:07.137807+0000","last_clean":"2023-10-09T04:12:07.137807+0000","last_became_active":"2023-10-09T04:11:26.220650+0000","last_became_peered":"2023-10-09T04:11:26.220650+0000","last_unstale":"2023-10-09T04:12:07.137807+0000","last_undegraded":"2023-10-09T04:12:07.137807+0000","last_fullsized":"2023-10-09T04:12:07.137807+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191961+0000","last_change":"2023-10-09T04:09:37.118061+0000","last_active":"2023-10-09T04:12:07.191961+0000","last_peered":"2023-10-09T04:12:07.191961+0000","last_clean":"2023-10-09T04:12:07.191961+0000","last_became_active":"2023-10-09T04:09:37.117163+0000","last_became_peered":"2023-10-09T04:09:37.117163+0000","last_unstale":"2023-10-09T04:12:07.191961+0000","last_undegraded":"2023-10-09T04:12:07.191961+0000","last_fullsized":"2023-10-09T04:12:07.191961+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133476+0000","last_change":"2023-10-09T04:11:26.239105+0000","last_active":"2023-10-09T04:12:07.133476+0000","last_peered":"2023-10-09T04:12:07.133476+0000","last_clean":"2023-10-09T04:12:07.133476+0000","last_became_active":"2023-10-09T04:11:26.238762+0000","last_became_peered":"2023-10-09T04:11:26.238762+0000","last_unstale":"2023-10-09T04:12:07.133476+0000","last_undegraded":"2023-10-09T04:12:07.133476+0000","last_fullsized":"2023-10-09T04:12:07.133476+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131270+0000","last_change":"2023-10-09T04:11:26.219084+0000","last_active":"2023-10-09T04:12:07.131270+0000","last_peered":"2023-10-09T04:12:07.131270+0000","last_clean":"2023-10-09T04:12:07.131270+0000","last_became_active":"2023-10-09T04:11:26.218727+0000","last_became_peered":"2023-10-09T04:11:26.218727+0000","last_unstale":"2023-10-09T04:12:07.131270+0000","last_undegraded":"2023-10-09T04:12:07.131270+0000","last_fullsized":"2023-10-09T04:12:07.131270+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191671+0000","last_change":"2023-10-09T04:09:37.117349+0000","last_active":"2023-10-09T04:12:07.191671+0000","last_peered":"2023-10-09T04:12:07.191671+0000","last_clean":"2023-10-09T04:12:07.191671+0000","last_became_active":"2023-10-09T04:09:37.116674+0000","last_became_peered":"2023-10-09T04:09:37.116674+0000","last_unstale":"2023-10-09T04:12:07.191671+0000","last_undegraded":"2023-10-09T04:12:07.191671+0000","last_fullsized":"2023-10-09T04:12:07.191671+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190792+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:12:07.190792+0000","last_peered":"2023-10-09T04:12:07.190792+0000","last_clean":"2023-10-09T04:12:07.190792+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:12:07.190792+0000","last_undegraded":"2023-10-09T04:12:07.190792+0000","last_fullsized":"2023-10-09T04:12:07.190792+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137799+0000","last_change":"2023-10-09T04:11:26.219926+0000","last_active":"2023-10-09T04:12:07.137799+0000","last_peered":"2023-10-09T04:12:07.137799+0000","last_clean":"2023-10-09T04:12:07.137799+0000","last_became_active":"2023-10-09T04:11:26.219597+0000","last_became_peered":"2023-10-09T04:11:26.219597+0000","last_unstale":"2023-10-09T04:12:07.137799+0000","last_undegraded":"2023-10-09T04:12:07.137799+0000","last_fullsized":"2023-10-09T04:12:07.137799+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191474+0000","last_change":"2023-10-09T04:08:38.365338+0000","last_active":"2023-10-09T04:12:07.191474+0000","last_peered":"2023-10-09T04:12:07.191474+0000","last_clean":"2023-10-09T04:12:07.191474+0000","last_became_active":"2023-10-09T04:08:38.364495+0000","last_became_peered":"2023-10-09T04:08:38.364495+0000","last_unstale":"2023-10-09T04:12:07.191474+0000","last_undegraded":"2023-10-09T04:12:07.191474+0000","last_fullsized":"2023-10-09T04:12:07.191474+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131752+0000","last_change":"2023-10-09T04:09:36.312522+0000","last_active":"2023-10-09T04:12:07.131752+0000","last_peered":"2023-10-09T04:12:07.131752+0000","last_clean":"2023-10-09T04:12:07.131752+0000","last_became_active":"2023-10-09T04:09:36.311979+0000","last_became_peered":"2023-10-09T04:09:36.311979+0000","last_unstale":"2023-10-09T04:12:07.131752+0000","last_undegraded":"2023-10-09T04:12:07.131752+0000","last_fullsized":"2023-10-09T04:12:07.131752+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138698+0000","last_change":"2023-10-09T04:11:26.839144+0000","last_active":"2023-10-09T04:12:07.138698+0000","last_peered":"2023-10-09T04:12:07.138698+0000","last_clean":"2023-10-09T04:12:07.138698+0000","last_became_active":"2023-10-09T04:11:26.838499+0000","last_became_peered":"2023-10-09T04:11:26.838499+0000","last_unstale":"2023-10-09T04:12:07.138698+0000","last_undegraded":"2023-10-09T04:12:07.138698+0000","last_fullsized":"2023-10-09T04:12:07.138698+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134937+0000","last_change":"2023-10-09T04:11:26.226361+0000","last_active":"2023-10-09T04:12:07.134937+0000","last_peered":"2023-10-09T04:12:07.134937+0000","last_clean":"2023-10-09T04:12:07.134937+0000","last_became_active":"2023-10-09T04:11:26.225698+0000","last_became_peered":"2023-10-09T04:11:26.225698+0000","last_unstale":"2023-10-09T04:12:07.134937+0000","last_undegraded":"2023-10-09T04:12:07.134937+0000","last_fullsized":"2023-10-09T04:12:07.134937+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134904+0000","last_change":"2023-10-09T04:09:37.118566+0000","last_active":"2023-10-09T04:12:07.134904+0000","last_peered":"2023-10-09T04:12:07.134904+0000","last_clean":"2023-10-09T04:12:07.134904+0000","last_became_active":"2023-10-09T04:09:37.117693+0000","last_became_peered":"2023-10-09T04:09:37.117693+0000","last_unstale":"2023-10-09T04:12:07.134904+0000","last_undegraded":"2023-10-09T04:12:07.134904+0000","last_fullsized":"2023-10-09T04:12:07.134904+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137639+0000","last_change":"2023-10-09T04:11:26.243218+0000","last_active":"2023-10-09T04:12:07.137639+0000","last_peered":"2023-10-09T04:12:07.137639+0000","last_clean":"2023-10-09T04:12:07.137639+0000","last_became_active":"2023-10-09T04:11:26.242904+0000","last_became_peered":"2023-10-09T04:11:26.242904+0000","last_unstale":"2023-10-09T04:12:07.137639+0000","last_undegraded":"2023-10-09T04:12:07.137639+0000","last_fullsized":"2023-10-09T04:12:07.137639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136193+0000","last_change":"2023-10-09T04:11:26.219069+0000","last_active":"2023-10-09T04:12:07.136193+0000","last_peered":"2023-10-09T04:12:07.136193+0000","last_clean":"2023-10-09T04:12:07.136193+0000","last_became_active":"2023-10-09T04:11:26.218722+0000","last_became_peered":"2023-10-09T04:11:26.218722+0000","last_unstale":"2023-10-09T04:12:07.136193+0000","last_undegraded":"2023-10-09T04:12:07.136193+0000","last_fullsized":"2023-10-09T04:12:07.136193+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131555+0000","last_change":"2023-10-09T04:09:36.309784+0000","last_active":"2023-10-09T04:12:07.131555+0000","last_peered":"2023-10-09T04:12:07.131555+0000","last_clean":"2023-10-09T04:12:07.131555+0000","last_became_active":"2023-10-09T04:09:36.309394+0000","last_became_peered":"2023-10-09T04:09:36.309394+0000","last_unstale":"2023-10-09T04:12:07.131555+0000","last_undegraded":"2023-10-09T04:12:07.131555+0000","last_fullsized":"2023-10-09T04:12:07.131555+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134672+0000","last_change":"2023-10-09T04:11:26.838943+0000","last_active":"2023-10-09T04:12:07.134672+0000","last_peered":"2023-10-09T04:12:07.134672+0000","last_clean":"2023-10-09T04:12:07.134672+0000","last_became_active":"2023-10-09T04:11:26.838306+0000","last_became_peered":"2023-10-09T04:11:26.838306+0000","last_unstale":"2023-10-09T04:12:07.134672+0000","last_undegraded":"2023-10-09T04:12:07.134672+0000","last_fullsized":"2023-10-09T04:12:07.134672+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131909+0000","last_change":"2023-10-09T04:11:26.224977+0000","last_active":"2023-10-09T04:12:07.131909+0000","last_peered":"2023-10-09T04:12:07.131909+0000","last_clean":"2023-10-09T04:12:07.131909+0000","last_became_active":"2023-10-09T04:11:26.224643+0000","last_became_peered":"2023-10-09T04:11:26.224643+0000","last_unstale":"2023-10-09T04:12:07.131909+0000","last_undegraded":"2023-10-09T04:12:07.131909+0000","last_fullsized":"2023-10-09T04:12:07.131909+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137480+0000","last_change":"2023-10-09T04:09:37.117854+0000","last_active":"2023-10-09T04:12:07.137480+0000","last_peered":"2023-10-09T04:12:07.137480+0000","last_clean":"2023-10-09T04:12:07.137480+0000","last_became_active":"2023-10-09T04:09:37.117379+0000","last_became_peered":"2023-10-09T04:09:37.117379+0000","last_unstale":"2023-10-09T04:12:07.137480+0000","last_undegraded":"2023-10-09T04:12:07.137480+0000","last_fullsized":"2023-10-09T04:12:07.137480+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138649+0000","last_change":"2023-10-09T04:11:26.242716+0000","last_active":"2023-10-09T04:12:07.138649+0000","last_peered":"2023-10-09T04:12:07.138649+0000","last_clean":"2023-10-09T04:12:07.138649+0000","last_became_active":"2023-10-09T04:11:26.242338+0000","last_became_peered":"2023-10-09T04:11:26.242338+0000","last_unstale":"2023-10-09T04:12:07.138649+0000","last_undegraded":"2023-10-09T04:12:07.138649+0000","last_fullsized":"2023-10-09T04:12:07.138649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137461+0000","last_change":"2023-10-09T04:11:26.222623+0000","last_active":"2023-10-09T04:12:07.137461+0000","last_peered":"2023-10-09T04:12:07.137461+0000","last_clean":"2023-10-09T04:12:07.137461+0000","last_became_active":"2023-10-09T04:11:26.222270+0000","last_became_peered":"2023-10-09T04:11:26.222270+0000","last_unstale":"2023-10-09T04:12:07.137461+0000","last_undegraded":"2023-10-09T04:12:07.137461+0000","last_fullsized":"2023-10-09T04:12:07.137461+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136471+0000","last_change":"2023-10-09T04:11:27.725832+0000","last_active":"2023-10-09T04:12:07.136471+0000","last_peered":"2023-10-09T04:12:07.136471+0000","last_clean":"2023-10-09T04:12:07.136471+0000","last_became_active":"2023-10-09T04:11:27.724932+0000","last_became_peered":"2023-10-09T04:11:27.724932+0000","last_unstale":"2023-10-09T04:12:07.136471+0000","last_undegraded":"2023-10-09T04:12:07.136471+0000","last_fullsized":"2023-10-09T04:12:07.136471+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135313+0000","last_change":"2023-10-09T04:11:26.214762+0000","last_active":"2023-10-09T04:12:07.135313+0000","last_peered":"2023-10-09T04:12:07.135313+0000","last_clean":"2023-10-09T04:12:07.135313+0000","last_became_active":"2023-10-09T04:11:26.214399+0000","last_became_peered":"2023-10-09T04:11:26.214399+0000","last_unstale":"2023-10-09T04:12:07.135313+0000","last_undegraded":"2023-10-09T04:12:07.135313+0000","last_fullsized":"2023-10-09T04:12:07.135313+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135865+0000","last_change":"2023-10-09T04:11:27.725640+0000","last_active":"2023-10-09T04:12:07.135865+0000","last_peered":"2023-10-09T04:12:07.135865+0000","last_clean":"2023-10-09T04:12:07.135865+0000","last_became_active":"2023-10-09T04:11:27.725109+0000","last_became_peered":"2023-10-09T04:11:27.725109+0000","last_unstale":"2023-10-09T04:12:07.135865+0000","last_undegraded":"2023-10-09T04:12:07.135865+0000","last_fullsized":"2023-10-09T04:12:07.135865+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191156+0000","last_change":"2023-10-09T04:11:26.217013+0000","last_active":"2023-10-09T04:12:07.191156+0000","last_peered":"2023-10-09T04:12:07.191156+0000","last_clean":"2023-10-09T04:12:07.191156+0000","last_became_active":"2023-10-09T04:11:26.216695+0000","last_became_peered":"2023-10-09T04:11:26.216695+0000","last_unstale":"2023-10-09T04:12:07.191156+0000","last_undegraded":"2023-10-09T04:12:07.191156+0000","last_fullsized":"2023-10-09T04:12:07.191156+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135772+0000","last_change":"2023-10-09T04:11:27.719835+0000","last_active":"2023-10-09T04:12:07.135772+0000","last_peered":"2023-10-09T04:12:07.135772+0000","last_clean":"2023-10-09T04:12:07.135772+0000","last_became_active":"2023-10-09T04:11:27.719092+0000","last_became_peered":"2023-10-09T04:11:27.719092+0000","last_unstale":"2023-10-09T04:12:07.135772+0000","last_undegraded":"2023-10-09T04:12:07.135772+0000","last_fullsized":"2023-10-09T04:12:07.135772+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138082+0000","last_change":"2023-10-09T04:11:26.225784+0000","last_active":"2023-10-09T04:12:07.138082+0000","last_peered":"2023-10-09T04:12:07.138082+0000","last_clean":"2023-10-09T04:12:07.138082+0000","last_became_active":"2023-10-09T04:11:26.225439+0000","last_became_peered":"2023-10-09T04:11:26.225439+0000","last_unstale":"2023-10-09T04:12:07.138082+0000","last_undegraded":"2023-10-09T04:12:07.138082+0000","last_fullsized":"2023-10-09T04:12:07.138082+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.296192+0000","last_change":"2023-10-09T04:12:08.296192+0000","last_active":"2023-10-09T04:12:08.296192+0000","last_peered":"2023-10-09T04:12:08.296192+0000","last_clean":"2023-10-09T04:12:08.296192+0000","last_became_active":"2023-10-09T04:12:08.295544+0000","last_became_peered":"2023-10-09T04:12:08.295544+0000","last_unstale":"2023-10-09T04:12:08.296192+0000","last_undegraded":"2023-10-09T04:12:08.296192+0000","last_fullsized":"2023-10-09T04:12:08.296192+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131582+0000","last_change":"2023-10-09T04:11:26.224909+0000","last_active":"2023-10-09T04:12:07.131582+0000","last_peered":"2023-10-09T04:12:07.131582+0000","last_clean":"2023-10-09T04:12:07.131582+0000","last_became_active":"2023-10-09T04:11:26.224492+0000","last_became_peered":"2023-10-09T04:11:26.224492+0000","last_unstale":"2023-10-09T04:12:07.131582+0000","last_undegraded":"2023-10-09T04:12:07.131582+0000","last_fullsized":"2023-10-09T04:12:07.131582+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190873+0000","last_change":"2023-10-09T04:11:27.723595+0000","last_active":"2023-10-09T04:12:07.190873+0000","last_peered":"2023-10-09T04:12:07.190873+0000","last_clean":"2023-10-09T04:12:07.190873+0000","last_became_active":"2023-10-09T04:11:27.723172+0000","last_became_peered":"2023-10-09T04:11:27.723172+0000","last_unstale":"2023-10-09T04:12:07.190873+0000","last_undegraded":"2023-10-09T04:12:07.190873+0000","last_fullsized":"2023-10-09T04:12:07.190873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192072+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:12:07.192072+0000","last_peered":"2023-10-09T04:12:07.192072+0000","last_clean":"2023-10-09T04:12:07.192072+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:12:07.192072+0000","last_undegraded":"2023-10-09T04:12:07.192072+0000","last_fullsized":"2023-10-09T04:12:07.192072+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135110+0000","last_change":"2023-10-09T04:11:27.726052+0000","last_active":"2023-10-09T04:12:07.135110+0000","last_peered":"2023-10-09T04:12:07.135110+0000","last_clean":"2023-10-09T04:12:07.135110+0000","last_became_active":"2023-10-09T04:11:27.724179+0000","last_became_peered":"2023-10-09T04:11:27.724179+0000","last_unstale":"2023-10-09T04:12:07.135110+0000","last_undegraded":"2023-10-09T04:12:07.135110+0000","last_fullsized":"2023-10-09T04:12:07.135110+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192146+0000","last_change":"2023-10-09T04:11:26.217513+0000","last_active":"2023-10-09T04:12:07.192146+0000","last_peered":"2023-10-09T04:12:07.192146+0000","last_clean":"2023-10-09T04:12:07.192146+0000","last_became_active":"2023-10-09T04:11:26.217186+0000","last_became_peered":"2023-10-09T04:11:26.217186+0000","last_unstale":"2023-10-09T04:12:07.192146+0000","last_undegraded":"2023-10-09T04:12:07.192146+0000","last_fullsized":"2023-10-09T04:12:07.192146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131421+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:12:07.131421+0000","last_peered":"2023-10-09T04:12:07.131421+0000","last_clean":"2023-10-09T04:12:07.131421+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:12:07.131421+0000","last_undegraded":"2023-10-09T04:12:07.131421+0000","last_fullsized":"2023-10-09T04:12:07.131421+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.192386+0000","last_change":"2023-10-09T04:11:26.218190+0000","last_active":"2023-10-09T04:12:07.192386+0000","last_peered":"2023-10-09T04:12:07.192386+0000","last_clean":"2023-10-09T04:12:07.192386+0000","last_became_active":"2023-10-09T04:11:26.217883+0000","last_became_peered":"2023-10-09T04:11:26.217883+0000","last_unstale":"2023-10-09T04:12:07.192386+0000","last_undegraded":"2023-10-09T04:12:07.192386+0000","last_fullsized":"2023-10-09T04:12:07.192386+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138240+0000","last_change":"2023-10-09T04:11:26.864299+0000","last_active":"2023-10-09T04:12:07.138240+0000","last_peered":"2023-10-09T04:12:07.138240+0000","last_clean":"2023-10-09T04:12:07.138240+0000","last_became_active":"2023-10-09T04:11:26.862910+0000","last_became_peered":"2023-10-09T04:11:26.862910+0000","last_unstale":"2023-10-09T04:12:07.138240+0000","last_undegraded":"2023-10-09T04:12:07.138240+0000","last_fullsized":"2023-10-09T04:12:07.138240+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190951+0000","last_change":"2023-10-09T04:11:26.216146+0000","last_active":"2023-10-09T04:12:07.190951+0000","last_peered":"2023-10-09T04:12:07.190951+0000","last_clean":"2023-10-09T04:12:07.190951+0000","last_became_active":"2023-10-09T04:11:26.215822+0000","last_became_peered":"2023-10-09T04:11:26.215822+0000","last_unstale":"2023-10-09T04:12:07.190951+0000","last_undegraded":"2023-10-09T04:12:07.190951+0000","last_fullsized":"2023-10-09T04:12:07.190951+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133252+0000","last_change":"2023-10-09T04:11:26.864729+0000","last_active":"2023-10-09T04:12:07.133252+0000","last_peered":"2023-10-09T04:12:07.133252+0000","last_clean":"2023-10-09T04:12:07.133252+0000","last_became_active":"2023-10-09T04:11:26.863731+0000","last_became_peered":"2023-10-09T04:11:26.863731+0000","last_unstale":"2023-10-09T04:12:07.133252+0000","last_undegraded":"2023-10-09T04:12:07.133252+0000","last_fullsized":"2023-10-09T04:12:07.133252+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.190704+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:12:07.190704+0000","last_peered":"2023-10-09T04:12:07.190704+0000","last_clean":"2023-10-09T04:12:07.190704+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:12:07.190704+0000","last_undegraded":"2023-10-09T04:12:07.190704+0000","last_fullsized":"2023-10-09T04:12:07.190704+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131629+0000","last_change":"2023-10-09T04:11:27.725820+0000","last_active":"2023-10-09T04:12:07.131629+0000","last_peered":"2023-10-09T04:12:07.131629+0000","last_clean":"2023-10-09T04:12:07.131629+0000","last_became_active":"2023-10-09T04:11:27.723296+0000","last_became_peered":"2023-10-09T04:11:27.723296+0000","last_unstale":"2023-10-09T04:12:07.131629+0000","last_undegraded":"2023-10-09T04:12:07.131629+0000","last_fullsized":"2023-10-09T04:12:07.131629+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131353+0000","last_change":"2023-10-09T04:11:26.223318+0000","last_active":"2023-10-09T04:12:07.131353+0000","last_peered":"2023-10-09T04:12:07.131353+0000","last_clean":"2023-10-09T04:12:07.131353+0000","last_became_active":"2023-10-09T04:11:26.222785+0000","last_became_peered":"2023-10-09T04:11:26.222785+0000","last_unstale":"2023-10-09T04:12:07.131353+0000","last_undegraded":"2023-10-09T04:12:07.131353+0000","last_fullsized":"2023-10-09T04:12:07.131353+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.138696+0000","last_change":"2023-10-09T04:11:26.862755+0000","last_active":"2023-10-09T04:12:07.138696+0000","last_peered":"2023-10-09T04:12:07.138696+0000","last_clean":"2023-10-09T04:12:07.138696+0000","last_became_active":"2023-10-09T04:11:26.862377+0000","last_became_peered":"2023-10-09T04:11:26.862377+0000","last_unstale":"2023-10-09T04:12:07.138696+0000","last_undegraded":"2023-10-09T04:12:07.138696+0000","last_fullsized":"2023-10-09T04:12:07.138696+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191221+0000","last_change":"2023-10-09T04:11:26.217595+0000","last_active":"2023-10-09T04:12:07.191221+0000","last_peered":"2023-10-09T04:12:07.191221+0000","last_clean":"2023-10-09T04:12:07.191221+0000","last_became_active":"2023-10-09T04:11:26.217263+0000","last_became_peered":"2023-10-09T04:11:26.217263+0000","last_unstale":"2023-10-09T04:12:07.191221+0000","last_undegraded":"2023-10-09T04:12:07.191221+0000","last_fullsized":"2023-10-09T04:12:07.191221+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.136186+0000","last_change":"2023-10-09T04:11:27.724916+0000","last_active":"2023-10-09T04:12:07.136186+0000","last_peered":"2023-10-09T04:12:07.136186+0000","last_clean":"2023-10-09T04:12:07.136186+0000","last_became_active":"2023-10-09T04:11:27.724494+0000","last_became_peered":"2023-10-09T04:11:27.724494+0000","last_unstale":"2023-10-09T04:12:07.136186+0000","last_undegraded":"2023-10-09T04:12:07.136186+0000","last_fullsized":"2023-10-09T04:12:07.136186+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135582+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:12:07.135582+0000","last_peered":"2023-10-09T04:12:07.135582+0000","last_clean":"2023-10-09T04:12:07.135582+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:12:07.135582+0000","last_undegraded":"2023-10-09T04:12:07.135582+0000","last_fullsized":"2023-10-09T04:12:07.135582+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134544+0000","last_change":"2023-10-09T04:11:26.368982+0000","last_active":"2023-10-09T04:12:07.134544+0000","last_peered":"2023-10-09T04:12:07.134544+0000","last_clean":"2023-10-09T04:12:07.134544+0000","last_became_active":"2023-10-09T04:11:26.368463+0000","last_became_peered":"2023-10-09T04:11:26.368463+0000","last_unstale":"2023-10-09T04:12:07.134544+0000","last_undegraded":"2023-10-09T04:12:07.134544+0000","last_fullsized":"2023-10-09T04:12:07.134544+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191861+0000","last_change":"2023-10-09T04:11:26.224975+0000","last_active":"2023-10-09T04:12:07.191861+0000","last_peered":"2023-10-09T04:12:07.191861+0000","last_clean":"2023-10-09T04:12:07.191861+0000","last_became_active":"2023-10-09T04:11:26.224674+0000","last_became_peered":"2023-10-09T04:11:26.224674+0000","last_unstale":"2023-10-09T04:12:07.191861+0000","last_undegraded":"2023-10-09T04:12:07.191861+0000","last_fullsized":"2023-10-09T04:12:07.191861+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137672+0000","last_change":"2023-10-09T04:11:26.863318+0000","last_active":"2023-10-09T04:12:07.137672+0000","last_peered":"2023-10-09T04:12:07.137672+0000","last_clean":"2023-10-09T04:12:07.137672+0000","last_became_active":"2023-10-09T04:11:26.862458+0000","last_became_peered":"2023-10-09T04:11:26.862458+0000","last_unstale":"2023-10-09T04:12:07.137672+0000","last_undegraded":"2023-10-09T04:12:07.137672+0000","last_fullsized":"2023-10-09T04:12:07.137672+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131894+0000","last_change":"2023-10-09T04:11:26.220111+0000","last_active":"2023-10-09T04:12:07.131894+0000","last_peered":"2023-10-09T04:12:07.131894+0000","last_clean":"2023-10-09T04:12:07.131894+0000","last_became_active":"2023-10-09T04:11:26.219798+0000","last_became_peered":"2023-10-09T04:11:26.219798+0000","last_unstale":"2023-10-09T04:12:07.131894+0000","last_undegraded":"2023-10-09T04:12:07.131894+0000","last_fullsized":"2023-10-09T04:12:07.131894+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137558+0000","last_change":"2023-10-09T04:11:26.368923+0000","last_active":"2023-10-09T04:12:07.137558+0000","last_peered":"2023-10-09T04:12:07.137558+0000","last_clean":"2023-10-09T04:12:07.137558+0000","last_became_active":"2023-10-09T04:11:26.368458+0000","last_became_peered":"2023-10-09T04:11:26.368458+0000","last_unstale":"2023-10-09T04:12:07.137558+0000","last_undegraded":"2023-10-09T04:12:07.137558+0000","last_fullsized":"2023-10-09T04:12:07.137558+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.131793+0000","last_change":"2023-10-09T04:11:26.216693+0000","last_active":"2023-10-09T04:12:07.131793+0000","last_peered":"2023-10-09T04:12:07.131793+0000","last_clean":"2023-10-09T04:12:07.131793+0000","last_became_active":"2023-10-09T04:11:26.216375+0000","last_became_peered":"2023-10-09T04:11:26.216375+0000","last_unstale":"2023-10-09T04:12:07.131793+0000","last_undegraded":"2023-10-09T04:12:07.131793+0000","last_fullsized":"2023-10-09T04:12:07.131793+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.191005+0000","last_change":"2023-10-09T04:11:27.721427+0000","last_active":"2023-10-09T04:12:07.191005+0000","last_peered":"2023-10-09T04:12:07.191005+0000","last_clean":"2023-10-09T04:12:07.191005+0000","last_became_active":"2023-10-09T04:11:27.720974+0000","last_became_peered":"2023-10-09T04:11:27.720974+0000","last_unstale":"2023-10-09T04:12:07.191005+0000","last_undegraded":"2023-10-09T04:12:07.191005+0000","last_fullsized":"2023-10-09T04:12:07.191005+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.137973+0000","last_change":"2023-10-09T04:11:26.221810+0000","last_active":"2023-10-09T04:12:07.137973+0000","last_peered":"2023-10-09T04:12:07.137973+0000","last_clean":"2023-10-09T04:12:07.137973+0000","last_became_active":"2023-10-09T04:11:26.221462+0000","last_became_peered":"2023-10-09T04:11:26.221462+0000","last_unstale":"2023-10-09T04:12:07.137973+0000","last_undegraded":"2023-10-09T04:12:07.137973+0000","last_fullsized":"2023-10-09T04:12:07.137973+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.133293+0000","last_change":"2023-10-09T04:11:26.866153+0000","last_active":"2023-10-09T04:12:07.133293+0000","last_peered":"2023-10-09T04:12:07.133293+0000","last_clean":"2023-10-09T04:12:07.133293+0000","last_became_active":"2023-10-09T04:11:26.865701+0000","last_became_peered":"2023-10-09T04:11:26.865701+0000","last_unstale":"2023-10-09T04:12:07.133293+0000","last_undegraded":"2023-10-09T04:12:07.133293+0000","last_fullsized":"2023-10-09T04:12:07.133293+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.134764+0000","last_change":"2023-10-09T04:11:26.227226+0000","last_active":"2023-10-09T04:12:07.134764+0000","last_peered":"2023-10-09T04:12:07.134764+0000","last_clean":"2023-10-09T04:12:07.134764+0000","last_became_active":"2023-10-09T04:11:26.226823+0000","last_became_peered":"2023-10-09T04:11:26.226823+0000","last_unstale":"2023-10-09T04:12:07.134764+0000","last_undegraded":"2023-10-09T04:12:07.134764+0000","last_fullsized":"2023-10-09T04:12:07.134764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:12:08.295233+0000","last_change":"2023-10-09T04:12:08.295233+0000","last_active":"2023-10-09T04:12:08.295233+0000","last_peered":"2023-10-09T04:12:08.295233+0000","last_clean":"2023-10-09T04:12:08.295233+0000","last_became_active":"2023-10-09T04:12:08.294781+0000","last_became_peered":"2023-10-09T04:12:08.294781+0000","last_unstale":"2023-10-09T04:12:08.295233+0000","last_undegraded":"2023-10-09T04:12:08.295233+0000","last_fullsized":"2023-10-09T04:12:08.295233+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T04:12:07.135167+0000","last_change":"2023-10-09T04:11:26.222061+0000","last_active":"2023-10-09T04:12:07.135167+0000","last_peered":"2023-10-09T04:12:07.135167+0000","last_clean":"2023-10-09T04:12:07.135167+0000","last_became_active":"2023-10-09T04:11:26.221709+0000","last_became_peered":"2023-10-09T04:11:26.221709+0000","last_unstale":"2023-10-09T04:12:07.135167+0000","last_undegraded":"2023-10-09T04:12:07.135167+0000","last_fullsized":"2023-10-09T04:12:07.135167+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430263,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.89000000000000001,"15min":0.88500000000000001},"min":{"1min":0.61599999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.218,"5min":1.25,"15min":1.25},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90400000000000003,"15min":0.91300000000000003},"min":{"1min":0.62,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2529999999999999,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.83199999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.872,"15min":0.84599999999999997},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.2270000000000001,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.95799999999999996,"15min":0.94399999999999995},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.4690000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1}]},{"osd":2,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.86499999999999999,"15min":0.88300000000000001},"min":{"1min":0.64100000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.097,"5min":1.425,"15min":1.425},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.92500000000000004,"15min":0.90500000000000003},"min":{"1min":0.58199999999999996,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2989999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.038}]},{"osd":3,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.90900000000000003,"15min":0.89200000000000002},"min":{"1min":0.74299999999999999,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.167,"5min":1.206,"15min":1.206},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.96499999999999997,"15min":0.94699999999999995},"min":{"1min":0.72799999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2470000000000001,"5min":2.165,"15min":2.165},"last":1.1279999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.89400000000000002,"15min":0.87},"min":{"1min":0.66200000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.216},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.91400000000000003,"15min":0.86199999999999999},"min":{"1min":0.748,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.296,"5min":1.349,"15min":1.349},"last":1.069}]},{"osd":5,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.95199999999999996,"15min":0.90900000000000003},"min":{"1min":0.69799999999999995,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.3109999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.995,"15min":0.95699999999999996},"min":{"1min":0.69099999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.5269999999999999,"5min":1.673,"15min":1.673},"last":1.1579999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.0289999999999999,"15min":0.98399999999999999},"min":{"1min":0.84899999999999998,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.569,"5min":1.591,"15min":1.591},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0800000000000001,"15min":1.1160000000000001},"min":{"1min":0.89100000000000001,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4510000000000001,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":1.109}]}]},{"osd":6,"up_from":42,"seq":180388626496,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297724,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447428,"statfs":{"total":95995035648,"available":95690166272,"internally_reserved":0,"allocated":643072,"data_stored":271554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.84599999999999997,"15min":0.83499999999999996},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.1850000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.57699999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.25,"5min":1.296,"15min":1.296},"last":1}]},{"osd":1,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.91100000000000003,"15min":0.93500000000000005},"min":{"1min":0.627,"5min":0.502,"15min":0.502},"max":{"1min":1.1739999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.93700000000000006,"15min":0.95799999999999996},"min":{"1min":0.77700000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.171,"5min":1.228,"15min":1.228},"last":0.96799999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.90300000000000002,"15min":0.88400000000000001},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.194,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.89600000000000002,"15min":0.91200000000000003},"min":{"1min":0.54100000000000004,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1319999999999999,"5min":1.242,"15min":1.242},"last":1.075}]},{"osd":3,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.85799999999999998,"15min":0.875},"min":{"1min":0.68999999999999995,"5min":0.496,"15min":0.496},"max":{"1min":1.2509999999999999,"5min":1.278,"15min":1.278},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.96299999999999997,"15min":0.96099999999999997},"min":{"1min":0.63,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.254,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.1100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.83499999999999996,"15min":0.79000000000000004},"min":{"1min":0.68700000000000006,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1719999999999999,"5min":1.329,"15min":1.329},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.86799999999999999,"15min":0.85599999999999998},"min":{"1min":0.66600000000000004,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2370000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.143}]},{"osd":5,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.94299999999999995,"15min":0.88600000000000001},"min":{"1min":0.78700000000000003,"5min":0.434,"15min":0.434},"max":{"1min":1.306,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.036,"5min":0.94099999999999995,"15min":0.92500000000000004},"min":{"1min":0.79700000000000004,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.224,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.038}]},{"osd":7,"last update":"Mon Oct 9 04:12:46 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0329999999999999,"15min":0.98499999999999999},"min":{"1min":0.86699999999999999,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.393,"5min":1.427,"15min":1.427},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.1060000000000001,"15min":1.056},"min":{"1min":0.94299999999999995,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.4870000000000001,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.254}]}]},{"osd":1,"up_from":13,"seq":55834574954,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67000000000000004,"15min":0.58099999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.84899999999999998,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.68999999999999995,"15min":0.60099999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.97399999999999998,"5min":1.149,"15min":1.149},"last":0.78800000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.72299999999999998,"15min":0.65300000000000002},"min":{"1min":0.38100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.048,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.72799999999999998,"15min":0.66500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.1799999999999999,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.81699999999999995,"15min":0.80100000000000005},"min":{"1min":0.51300000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0649999999999999,"5min":1.151,"15min":1.151},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.85699999999999998,"15min":0.81799999999999995},"min":{"1min":0.46600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.218,"5min":1.258,"15min":1.258},"last":0.88400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.91300000000000003,"15min":0.92100000000000004},"min":{"1min":0.65900000000000003,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.254,"5min":1.347,"15min":1.347},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.0069999999999999,"15min":0.98699999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"max":{"1min":1.391,"5min":1.641,"15min":1.641},"last":1.123}]},{"osd":5,"last update":"Mon Oct 9 04:12:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.98099999999999998,"15min":1.0089999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.353,"5min":1.421,"15min":1.421},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.97199999999999998,"15min":1.012},"min":{"1min":0.72199999999999998,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2669999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.97899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.032,"15min":1.0349999999999999},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3080000000000001,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.997},{"interface":"front","average":{"1min":1.048,"5min":1.032,"15min":1.0329999999999999},"min":{"1min":0.69299999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.2370000000000001,"5min":3.327,"15min":3.327},"last":1.0589999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.0109999999999999,"15min":1.0329999999999999},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.2889999999999999,"5min":1.502,"15min":1.502},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0760000000000001,"15min":1.1100000000000001},"min":{"1min":0.69999999999999996,"5min":0.65900000000000003,"15min":0.65900000000000003},"max":{"1min":1.464,"5min":1.706,"15min":1.706},"last":1.0149999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738482,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.65000000000000002,"15min":0.59199999999999997},"min":{"1min":0.38200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.873,"5min":1.012,"15min":1.012},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.626,"5min":0.69999999999999996,"15min":0.626},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.94499999999999995,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.76700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.71699999999999997,"15min":0.66200000000000003},"min":{"1min":0.42799999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.86799999999999999,"5min":1.444,"15min":1.444},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.73699999999999999,"15min":0.67700000000000005},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.0860000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.85499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.73299999999999998,"15min":0.73499999999999999},"min":{"1min":0.44,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.1240000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80600000000000005},"min":{"1min":0.49299999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.2270000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.90100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.98199999999999998,"15min":0.97499999999999998},"min":{"1min":0.65500000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.468,"5min":1.506,"15min":1.506},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.98899999999999999,"15min":0.96299999999999997},"min":{"1min":0.56100000000000005,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":1.3140000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.042,"15min":1.0449999999999999},"min":{"1min":0.61799999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.4690000000000001,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.0449999999999999},{"interface":"front","average":{"1min":1,"5min":1.0169999999999999,"15min":1.012},"min":{"1min":0.71199999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":1.3979999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:12:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0589999999999999,"15min":1.087},"min":{"1min":0.73799999999999999,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.123,"15min":1.163},"min":{"1min":0.79200000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":1.1160000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.071,"15min":1.089},"min":{"1min":0.71999999999999997,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.675,"5min":1.675,"15min":1.675},"last":1.254},{"interface":"front","average":{"1min":1.077,"5min":1.129,"15min":1.149},"min":{"1min":0.77300000000000002,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.4199999999999999,"5min":1.448,"15min":1.448},"last":1.1759999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411426,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.65800000000000003,"15min":0.60499999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.745,"5min":0.71499999999999997,"15min":0.64800000000000002},"min":{"1min":0.40600000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.90800000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.72599999999999998,"15min":0.66400000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.73699999999999999,"15min":0.68300000000000005},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.73499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:03 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.74299999999999999,"15min":0.76700000000000002},"min":{"1min":0.57299999999999995,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.1240000000000001,"5min":1.137,"15min":1.137},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.79000000000000004,"15min":0.80100000000000005},"min":{"1min":0.47499999999999998,"5min":0.435,"15min":0.435},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.95499999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.95699999999999996,"15min":0.93500000000000005},"min":{"1min":0.58099999999999996,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2809999999999999,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.02,"5min":0.997,"15min":0.95999999999999996},"min":{"1min":0.70999999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.423,"5min":1.468,"15min":1.468},"last":0.85399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.008},"min":{"1min":0.71099999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.02,"15min":1.0209999999999999},"min":{"1min":0.75600000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.83099999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:12:03 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0620000000000001,"15min":1.044},"min":{"1min":0.86199999999999999,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.663,"5min":1.663,"15min":1.663},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.0840000000000001,"15min":1.093},"min":{"1min":0.93899999999999995,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.736,"5min":1.736,"15min":1.736},"last":1.05}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0900000000000001,"15min":1.1100000000000001},"min":{"1min":0.63,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.385,"5min":1.619,"15min":1.619},"last":1.115},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.087,"15min":1.069},"min":{"1min":0.76500000000000001,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.401,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":1.1830000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182487,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.69999999999999996},"min":{"1min":0.44600000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.65200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.69699999999999995,"15min":0.70599999999999996},"min":{"1min":0.37,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.95999999999999996,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.77400000000000002,"15min":0.77600000000000002},"min":{"1min":0.47099999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.1160000000000001,"5min":1.268,"15min":1.4910000000000001},"last":0.72299999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:12:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.432,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.218,"5min":1.597,"15min":1.597},"last":0.995},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.95599999999999996,"5min":1.522,"15min":1.522},"last":0.77600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.92200000000000004,"15min":0.90000000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.173,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0189999999999999,"15min":1.0229999999999999},"min":{"1min":0.67400000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.2969999999999999,"5min":1.482,"15min":1.482},"last":0.91500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.97899999999999998,"15min":1.0169999999999999},"min":{"1min":0.63700000000000001,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.4039999999999999,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.052},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0680000000000001,"15min":1.0860000000000001},"min":{"1min":0.84999999999999998,"5min":0.624,"15min":0.624},"max":{"1min":1.351,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.125}]},{"osd":6,"last update":"Mon Oct 9 04:12:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0569999999999999,"15min":1.0609999999999999},"min":{"1min":0.59999999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3799999999999999,"5min":1.583,"15min":1.583},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.119,"15min":1.127},"min":{"1min":0.82499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.4159999999999999,"5min":1.552,"15min":1.552},"last":1.0760000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:46 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0509999999999999,"15min":1.1000000000000001},"min":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.2669999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0840000000000001,"15min":1.1060000000000001},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.454,"5min":1.621,"15min":1.621},"last":1.1060000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018960,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.85799999999999998,"15min":0.83599999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.159,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91200000000000003,"15min":0.86399999999999999},"min":{"1min":0.59799999999999998,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.1299999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.77600000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84999999999999998,"15min":0.83899999999999997},"min":{"1min":0.58599999999999997,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.137,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.88300000000000001,"15min":0.83699999999999997},"min":{"1min":0.63900000000000001,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.087,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.97299999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.89600000000000002,"15min":0.86499999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1140000000000001,"5min":1.409,"15min":1.409},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.93400000000000005,"15min":0.89400000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.169,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.0309999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.93899999999999995,"15min":0.88100000000000001},"min":{"1min":0.71999999999999997,"5min":0.621,"15min":0.621},"max":{"1min":1.206,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.181},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.88400000000000001,"15min":0.88900000000000001},"min":{"1min":0.68300000000000005,"5min":0.60199999999999998,"15min":0.58099999999999996},"max":{"1min":1.044,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.92600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.84799999999999998,"15min":0.77800000000000002},"min":{"1min":0.52500000000000002,"5min":0.433,"15min":0.433},"max":{"1min":1.284,"5min":1.579,"15min":1.579},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.82799999999999996,"15min":0.73099999999999998},"min":{"1min":0.60899999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.147,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.72399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.88,"15min":0.879},"min":{"1min":0.65800000000000003,"5min":0.496,"15min":0.496},"max":{"1min":1.157,"5min":1.609,"15min":1.609},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.02,"5min":0.95899999999999996,"15min":0.93700000000000006},"min":{"1min":0.48999999999999999,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.5009999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.1100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.099,"15min":1.119},"min":{"1min":0.65900000000000003,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.353,"5min":1.653,"15min":1.653},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.042,"15min":1.056},"min":{"1min":0.73999999999999999,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2310000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.84399999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822727,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.85899999999999999,"15min":0.84099999999999997},"min":{"1min":0.65700000000000003,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.246,"5min":1.246,"15min":1.246},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86699999999999999,"15min":0.84499999999999997},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.153,"5min":1.232,"15min":1.232},"last":1.1819999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.64400000000000002,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.093,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.879,"15min":0.85799999999999998},"min":{"1min":0.55600000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":1.137}]},{"osd":2,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.88800000000000001,"15min":0.88},"min":{"1min":0.72299999999999998,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.361,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.056},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.89400000000000002,"15min":0.88200000000000001},"min":{"1min":0.56999999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.1679999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.90600000000000003,"15min":0.89000000000000001},"min":{"1min":0.53900000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.95499999999999996,"15min":0.92300000000000004},"min":{"1min":0.74199999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2310000000000001,"5min":1.458,"15min":1.458},"last":1.026}]},{"osd":4,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.86799999999999999,"15min":0.79200000000000004},"min":{"1min":0.52300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2869999999999999,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.097,"5min":0.92800000000000005,"15min":0.877},"min":{"1min":0.49299999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.78500000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:12:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.876,"15min":0.77800000000000002},"min":{"1min":0.47799999999999998,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.131,"5min":0.93899999999999995,"15min":0.88100000000000001},"min":{"1min":0.60899999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":0.92500000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:12:41 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.0209999999999999,"15min":0.92100000000000004},"min":{"1min":0.68899999999999995,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.4490000000000001,"5min":1.486,"15min":1.486},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.056,"15min":0.96399999999999997},"min":{"1min":0.627,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.357,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.087}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:13:01.665 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:13:01.665 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:13:01.665 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:13:01.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:13:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:02 smithi153 bash[27868]: audit 2023-10-09T04:13:00.874699+0000 mgr.y (mgr.14436) 151 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:02 smithi153 bash[27868]: cluster 2023-10-09T04:13:00.889898+0000 mgr.y (mgr.14436) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:02.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:02 smithi082 bash[27462]: audit 2023-10-09T04:13:00.874699+0000 mgr.y (mgr.14436) 151 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:02.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:02 smithi082 bash[27462]: cluster 2023-10-09T04:13:00.889898+0000 mgr.y (mgr.14436) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:02 smithi082 bash[33901]: audit 2023-10-09T04:13:00.874699+0000 mgr.y (mgr.14436) 151 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:02 smithi082 bash[33901]: cluster 2023-10-09T04:13:00.889898+0000 mgr.y (mgr.14436) 152 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:03.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:02] "GET /metrics HTTP/1.1" 200 191381 "" "Prometheus/2.33.4" 2023-10-09T04:13:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:04 smithi153 bash[27868]: cluster 2023-10-09T04:13:02.890932+0000 mgr.y (mgr.14436) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:04.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:03 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:04 smithi082 bash[27462]: cluster 2023-10-09T04:13:02.890932+0000 mgr.y (mgr.14436) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:04 smithi082 bash[33901]: cluster 2023-10-09T04:13:02.890932+0000 mgr.y (mgr.14436) 153 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:06 smithi153 bash[27868]: cluster 2023-10-09T04:13:04.891561+0000 mgr.y (mgr.14436) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:06.480 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:06.662 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:06 smithi082 bash[27462]: cluster 2023-10-09T04:13:04.891561+0000 mgr.y (mgr.14436) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:06.662 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:06 smithi082 bash[33901]: cluster 2023-10-09T04:13:04.891561+0000 mgr.y (mgr.14436) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:08 smithi153 bash[27868]: cluster 2023-10-09T04:13:06.892212+0000 mgr.y (mgr.14436) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:08.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:08 smithi082 bash[27462]: cluster 2023-10-09T04:13:06.892212+0000 mgr.y (mgr.14436) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:08.597 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:08 smithi082 bash[33901]: cluster 2023-10-09T04:13:06.892212+0000 mgr.y (mgr.14436) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:09.333 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:13:09.986 INFO:teuthology.orchestra.run.smithi082.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:13:10.092 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:13:10.093 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:13:10.106 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T04:13:10.207 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:13:10.220 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:13:10.221 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:13:10.222 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:13:10.227 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T04:13:10.358 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:13:10.363 DEBUG:teuthology.orchestra.run.smithi082:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:13:10.364 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:13:10.364 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T04:13:10.364 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi082.front.sepia.ceph.com 2023-10-09T04:13:10.365 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T04:13:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:13:10.484 INFO:teuthology.orchestra.run.smithi082.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:13:10.485 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:10 smithi082 bash[27462]: cluster 2023-10-09T04:13:08.893545+0000 mgr.y (mgr.14436) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:10 smithi082 bash[27462]: audit 2023-10-09T04:13:09.330761+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.82:0/4185001394' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:10 smithi082 bash[33901]: cluster 2023-10-09T04:13:08.893545+0000 mgr.y (mgr.14436) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:10 smithi082 bash[33901]: audit 2023-10-09T04:13:09.330761+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.82:0/4185001394' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:13:10.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:10 smithi153 bash[27868]: cluster 2023-10-09T04:13:08.893545+0000 mgr.y (mgr.14436) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:10.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:10 smithi153 bash[27868]: audit 2023-10-09T04:13:09.330761+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.82:0/4185001394' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:13:10.533 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:13:10.598 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:13:10.606 INFO:tasks.workunit:timeout=3h 2023-10-09T04:13:10.606 INFO:tasks.workunit:cleanup=True 2023-10-09T04:13:10.607 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:13:10.658 INFO:tasks.workunit.client.0.smithi082.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T04:13:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:12 smithi153 bash[27868]: cluster 2023-10-09T04:13:10.894145+0000 mgr.y (mgr.14436) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:13.977 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:12 smithi082 bash[33901]: cluster 2023-10-09T04:13:10.894145+0000 mgr.y (mgr.14436) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:13.977 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:12 smithi082 bash[27462]: cluster 2023-10-09T04:13:10.894145+0000 mgr.y (mgr.14436) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:13.977 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:12] "GET /metrics HTTP/1.1" 200 191392 "" "Prometheus/2.33.4" 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE Bus STOPPING 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE Bus STOPPED 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE Bus STARTING 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE Serving on http://:::9283 2023-10-09T04:13:13.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:13 smithi082 bash[27740]: [09/Oct/2023:04:13:13] ENGINE Bus STARTED 2023-10-09T04:13:14.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T04:13:14.603 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:13 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:14.603 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[33901]: cluster 2023-10-09T04:13:12.895411+0000 mgr.y (mgr.14436) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:14.603 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[33901]: audit 2023-10-09T04:13:13.366773+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.82:0/3618861763' entity='client.admin' 2023-10-09T04:13:14.603 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[33901]: audit 2023-10-09T04:13:13.406823+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:14 smithi153 bash[27868]: cluster 2023-10-09T04:13:12.895411+0000 mgr.y (mgr.14436) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:14 smithi153 bash[27868]: audit 2023-10-09T04:13:13.366773+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.82:0/3618861763' entity='client.admin' 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:14 smithi153 bash[27868]: audit 2023-10-09T04:13:13.406823+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[27462]: cluster 2023-10-09T04:13:12.895411+0000 mgr.y (mgr.14436) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[27462]: audit 2023-10-09T04:13:13.366773+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.82:0/3618861763' entity='client.admin' 2023-10-09T04:13:14.604 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:14 smithi082 bash[27462]: audit 2023-10-09T04:13:13.406823+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:15.233 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T04:13:15.809 INFO:tasks.workunit.client.0.smithi082.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T04:13:15.852 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:13:15.852 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T04:13:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:16 smithi153 bash[27868]: cluster 2023-10-09T04:13:14.896026+0000 mgr.y (mgr.14436) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:16 smithi082 bash[27462]: cluster 2023-10-09T04:13:14.896026+0000 mgr.y (mgr.14436) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:16.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:16 smithi082 bash[33901]: cluster 2023-10-09T04:13:14.896026+0000 mgr.y (mgr.14436) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:18 smithi082 bash[27462]: cluster 2023-10-09T04:13:16.896472+0000 mgr.y (mgr.14436) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:18 smithi082 bash[33901]: cluster 2023-10-09T04:13:16.896472+0000 mgr.y (mgr.14436) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:18 smithi153 bash[27868]: cluster 2023-10-09T04:13:16.896472+0000 mgr.y (mgr.14436) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:19.725 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T04:13:19.824 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T04:13:19.881 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:13:19.883 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:13:19.884 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T04:13:19.940 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:13:19.946 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:13:19.946 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T04:13:19.953 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T04:13:19.954 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T04:13:19.954 DEBUG:teuthology.orchestra.run.smithi082:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T04:13:19.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: cluster 2023-10-09T04:13:18.897236+0000 mgr.y (mgr.14436) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:19.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: audit 2023-10-09T04:13:18.954366+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:19.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: audit 2023-10-09T04:13:19.320839+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:19.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: audit 2023-10-09T04:13:19.322420+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:19.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: audit 2023-10-09T04:13:19.323681+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:19.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27462]: audit 2023-10-09T04:13:19.332049+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:19.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T04:13:19.971 INFO:tasks.workunit.client.0.smithi082.stderr:+ TMPDIR=/tmp/rbd_import_export_65968 2023-10-09T04:13:19.971 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm -rf /tmp/rbd_import_export_65968 2023-10-09T04:13:19.971 INFO:tasks.workunit.client.0.smithi082.stderr:+ mkdir /tmp/rbd_import_export_65968 2023-10-09T04:13:19.972 INFO:tasks.workunit.client.0.smithi082.stderr:+ trap rm -rf /tmp/rbd_import_export_65968 INT TERM EXIT 2023-10-09T04:13:19.972 INFO:tasks.workunit.client.0.smithi082.stderr:+ mkdir foo.65968 2023-10-09T04:13:19.973 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import foo.65968 foo.dir 2023-10-09T04:13:20.015 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: cannot import a directory 2023-10-09T04:13:20.015 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 0% complete...failed. 2023-10-09T04:13:20.015 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: import failed: (21) Is a directory 2023-10-09T04:13:20.019 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:20.019 INFO:tasks.workunit.client.0.smithi082.stderr:+ rmdir foo.65968 2023-10-09T04:13:20.053 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_65968/img bs=1k count=1 seek=10 2023-10-09T04:13:20.055 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:13:20.055 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:13:20.055 INFO:tasks.workunit.client.0.smithi082.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000180919 s, 5.7 MB/s 2023-10-09T04:13:20.056 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_65968/img bs=1k count=10 seek=100 2023-10-09T04:13:20.056 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records in 2023-10-09T04:13:20.056 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records out 2023-10-09T04:13:20.056 INFO:tasks.workunit.client.0.smithi082.stderr:10240 bytes (10 kB, 10 KiB) copied, 9.4593e-05 s, 108 MB/s 2023-10-09T04:13:20.056 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_65968/img bs=1k count=100 seek=1000 2023-10-09T04:13:20.057 INFO:tasks.workunit.client.0.smithi082.stderr:70+1 records in 2023-10-09T04:13:20.057 INFO:tasks.workunit.client.0.smithi082.stderr:70+1 records out 2023-10-09T04:13:20.057 INFO:tasks.workunit.client.0.smithi082.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000535489 s, 135 MB/s 2023-10-09T04:13:20.057 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_65968/img bs=1k seek=10000 2023-10-09T04:13:20.059 INFO:tasks.workunit.client.0.smithi082.stderr:138+1 records in 2023-10-09T04:13:20.059 INFO:tasks.workunit.client.0.smithi082.stderr:138+1 records out 2023-10-09T04:13:20.059 INFO:tasks.workunit.client.0.smithi082.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000394114 s, 361 MB/s 2023-10-09T04:13:20.059 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_65968/img bs=1k seek=100000 2023-10-09T04:13:20.059 INFO:tasks.workunit.client.0.smithi082.stderr:74+1 records in 2023-10-09T04:13:20.060 INFO:tasks.workunit.client.0.smithi082.stderr:74+1 records out 2023-10-09T04:13:20.060 INFO:tasks.workunit.client.0.smithi082.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.000251771 s, 302 MB/s 2023-10-09T04:13:20.060 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_65968/img bs=1k seek=1000000 2023-10-09T04:13:20.061 INFO:tasks.workunit.client.0.smithi082.stderr:194+1 records in 2023-10-09T04:13:20.061 INFO:tasks.workunit.client.0.smithi082.stderr:194+1 records out 2023-10-09T04:13:20.061 INFO:tasks.workunit.client.0.smithi082.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.000526909 s, 378 MB/s 2023-10-09T04:13:20.061 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:13:20.112 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:13:20.112 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:13:20.117 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:20.117 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: cluster 2023-10-09T04:13:18.897236+0000 mgr.y (mgr.14436) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: audit 2023-10-09T04:13:18.954366+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: audit 2023-10-09T04:13:19.320839+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: audit 2023-10-09T04:13:19.322420+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: audit 2023-10-09T04:13:19.323681+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[33901]: audit 2023-10-09T04:13:19.332049+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27740]: [09/Oct/2023:04:13:19] ENGINE Bus STOPPING 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27740]: [09/Oct/2023:04:13:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27740]: [09/Oct/2023:04:13:19] ENGINE Bus STOPPED 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:19 smithi082 bash[27740]: [09/Oct/2023:04:13:19] ENGINE Bus STARTING 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27740]: [09/Oct/2023:04:13:20] ENGINE Serving on http://:::9283 2023-10-09T04:13:20.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27740]: [09/Oct/2023:04:13:20] ENGINE Bus STARTED 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: cluster 2023-10-09T04:13:18.897236+0000 mgr.y (mgr.14436) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: audit 2023-10-09T04:13:18.954366+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: audit 2023-10-09T04:13:19.320839+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: audit 2023-10-09T04:13:19.322420+0000 mon.a (mon.0) 729 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: audit 2023-10-09T04:13:19.323681+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:19 smithi153 bash[27868]: audit 2023-10-09T04:13:19.332049+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:20.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T04:13:21.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27462]: audit 2023-10-09T04:13:19.954795+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T04:13:21.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27462]: audit 2023-10-09T04:13:20.003319+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:21.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27462]: audit 2023-10-09T04:13:20.005485+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27462]: audit 2023-10-09T04:13:20.006542+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[27462]: audit 2023-10-09T04:13:20.015336+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[33901]: audit 2023-10-09T04:13:19.954795+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[33901]: audit 2023-10-09T04:13:20.003319+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[33901]: audit 2023-10-09T04:13:20.005485+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[33901]: audit 2023-10-09T04:13:20.006542+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:21.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:20 smithi082 bash[33901]: audit 2023-10-09T04:13:20.015336+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:20 smithi153 bash[27868]: audit 2023-10-09T04:13:19.954795+0000 mon.a (mon.0) 732 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T04:13:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:20 smithi153 bash[27868]: audit 2023-10-09T04:13:20.003319+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:20 smithi153 bash[27868]: audit 2023-10-09T04:13:20.005485+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:20 smithi153 bash[27868]: audit 2023-10-09T04:13:20.006542+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:20 smithi153 bash[27868]: audit 2023-10-09T04:13:20.015336+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:22 smithi082 bash[33901]: cluster 2023-10-09T04:13:20.897897+0000 mgr.y (mgr.14436) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:22 smithi082 bash[33901]: cluster 2023-10-09T04:13:21.023016+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:13:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:22 smithi082 bash[27462]: cluster 2023-10-09T04:13:20.897897+0000 mgr.y (mgr.14436) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:22.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:22 smithi082 bash[27462]: cluster 2023-10-09T04:13:21.023016+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:13:22.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:22 smithi153 bash[27868]: cluster 2023-10-09T04:13:20.897897+0000 mgr.y (mgr.14436) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:22 smithi153 bash[27868]: cluster 2023-10-09T04:13:21.023016+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:13:22.655 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T04:13:22.656 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:22.661 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg /tmp/rbd_import_export_65968/img2 2023-10-09T04:13:22.985 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:22.990 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg - 2023-10-09T04:13:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:23 smithi082 bash[27462]: cluster 2023-10-09T04:13:22.038068+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:13:23.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:22] "GET /metrics HTTP/1.1" 200 191401 "" "Prometheus/2.33.4" 2023-10-09T04:13:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:23 smithi082 bash[33901]: cluster 2023-10-09T04:13:22.038068+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:13:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:23 smithi153 bash[27868]: cluster 2023-10-09T04:13:22.038068+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:13:23.861 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:23.867 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:13:24.002 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:13:24.007 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img2 2023-10-09T04:13:24.085 INFO:teuthology.orchestra.run.smithi082.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:24 smithi153 bash[27868]: cluster 2023-10-09T04:13:22.898654+0000 mgr.y (mgr.14436) 163 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 64 KiB/s wr, 14 op/s 2023-10-09T04:13:24.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:23 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:24 smithi082 bash[27462]: cluster 2023-10-09T04:13:22.898654+0000 mgr.y (mgr.14436) 163 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 64 KiB/s wr, 14 op/s 2023-10-09T04:13:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:24 smithi082 bash[33901]: cluster 2023-10-09T04:13:22.898654+0000 mgr.y (mgr.14436) 163 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 64 KiB/s wr, 14 op/s 2023-10-09T04:13:24.749 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img3 2023-10-09T04:13:25.271 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img2 /tmp/rbd_import_export_65968/img3 2023-10-09T04:13:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.074134+0000 mgr.y (mgr.14436) 164 : audit [DBG] from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: cephadm 2023-10-09T04:13:24.075652+0000 mgr.y (mgr.14436) 165 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.082909+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: cluster 2023-10-09T04:13:24.090795+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:13:25.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.097303+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:25.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.099187+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:25.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.100149+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:25.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: audit 2023-10-09T04:13:24.106920+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:25 smithi153 bash[27868]: cephadm 2023-10-09T04:13:24.113576+0000 mgr.y (mgr.14436) 166 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:25.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.074134+0000 mgr.y (mgr.14436) 164 : audit [DBG] from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: cephadm 2023-10-09T04:13:24.075652+0000 mgr.y (mgr.14436) 165 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.082909+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: cluster 2023-10-09T04:13:24.090795+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.097303+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.099187+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.100149+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: audit 2023-10-09T04:13:24.106920+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[27462]: cephadm 2023-10-09T04:13:24.113576+0000 mgr.y (mgr.14436) 166 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:25.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.074134+0000 mgr.y (mgr.14436) 164 : audit [DBG] from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: cephadm 2023-10-09T04:13:24.075652+0000 mgr.y (mgr.14436) 165 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.082909+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: cluster 2023-10-09T04:13:24.090795+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.097303+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.099187+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.100149+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: audit 2023-10-09T04:13:24.106920+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:13:25.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:25 smithi082 bash[33901]: cephadm 2023-10-09T04:13:24.113576+0000 mgr.y (mgr.14436) 166 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:13:26.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:26 smithi153 bash[27868]: cluster 2023-10-09T04:13:24.899054+0000 mgr.y (mgr.14436) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:26.487 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:26 smithi082 bash[27462]: cluster 2023-10-09T04:13:24.899054+0000 mgr.y (mgr.14436) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:26.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:26 smithi082 bash[33901]: cluster 2023-10-09T04:13:24.899054+0000 mgr.y (mgr.14436) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:28 smithi153 bash[27868]: cluster 2023-10-09T04:13:26.899700+0000 mgr.y (mgr.14436) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:28.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:28 smithi082 bash[33901]: cluster 2023-10-09T04:13:26.899700+0000 mgr.y (mgr.14436) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:28.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:28 smithi082 bash[27462]: cluster 2023-10-09T04:13:26.899700+0000 mgr.y (mgr.14436) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 85 KiB/s wr, 19 op/s 2023-10-09T04:13:30.389 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format - testimg 2023-10-09T04:13:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:30 smithi153 bash[27868]: cluster 2023-10-09T04:13:28.900704+0000 mgr.y (mgr.14436) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 66 KiB/s wr, 38 op/s 2023-10-09T04:13:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:30 smithi082 bash[27462]: cluster 2023-10-09T04:13:28.900704+0000 mgr.y (mgr.14436) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 66 KiB/s wr, 38 op/s 2023-10-09T04:13:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:30 smithi082 bash[33901]: cluster 2023-10-09T04:13:28.900704+0000 mgr.y (mgr.14436) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 66 KiB/s wr, 38 op/s 2023-10-09T04:13:30.872 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:30.872 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:30.877 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg /tmp/rbd_import_export_65968/img2 2023-10-09T04:13:31.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:13:31.213 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:31.217 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg - 2023-10-09T04:13:32.124 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:32.130 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:13:32.258 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:13:32.262 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img2 2023-10-09T04:13:32.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:32 smithi153 bash[27868]: cluster 2023-10-09T04:13:30.901167+0000 mgr.y (mgr.14436) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-10-09T04:13:32.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:32 smithi082 bash[27462]: cluster 2023-10-09T04:13:30.901167+0000 mgr.y (mgr.14436) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-10-09T04:13:32.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:32 smithi082 bash[33901]: cluster 2023-10-09T04:13:30.901167+0000 mgr.y (mgr.14436) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 59 KiB/s wr, 34 op/s 2023-10-09T04:13:33.023 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img3 2023-10-09T04:13:33.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:32] "GET /metrics HTTP/1.1" 200 191401 "" "Prometheus/2.33.4" 2023-10-09T04:13:33.559 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img2 /tmp/rbd_import_export_65968/img3 2023-10-09T04:13:34.333 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep -q export-format 2023-10-09T04:13:34.334 INFO:tasks.workunit.client.0.smithi082.stderr:rbd help export 2023-10-09T04:13:34.350 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_65968/img bs=1k count=10 seek=100 2023-10-09T04:13:34.351 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records in 2023-10-09T04:13:34.351 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records out 2023-10-09T04:13:34.351 INFO:tasks.workunit.client.0.smithi082.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.0001023 s, 100 MB/s 2023-10-09T04:13:34.351 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:34.418 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:34.419 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:34.421 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap 2023-10-09T04:13:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:34 smithi153 bash[27868]: cluster 2023-10-09T04:13:32.901942+0000 mgr.y (mgr.14436) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 54 KiB/s wr, 68 op/s 2023-10-09T04:13:34.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:34.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:34 smithi082 bash[27462]: cluster 2023-10-09T04:13:32.901942+0000 mgr.y (mgr.14436) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 54 KiB/s wr, 68 op/s 2023-10-09T04:13:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:34 smithi082 bash[33901]: cluster 2023-10-09T04:13:32.901942+0000 mgr.y (mgr.14436) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 54 KiB/s wr, 68 op/s 2023-10-09T04:13:35.149 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:13:35.156 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T04:13:35.218 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta list testimg 2023-10-09T04:13:35.266 INFO:tasks.workunit.client.0.smithi082.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T04:13:35.266 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:13:35.266 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:13:35.266 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 2023-10-09T04:13:35.266 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:35.314 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:35.319 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65968/img_v2 testimg_import 2023-10-09T04:13:36.174 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:13:36.179 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:13:36.221 INFO:tasks.workunit.client.0.smithi082.stdout:rbd image 'testimg_import': 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: size 110 KiB in 1 objects 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: order 22 (4 MiB objects) 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: snapshot_count: 1 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: id: 3a381dd34b2f 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: block_name_prefix: rbd_data.3a381dd34b2f 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: format: 2 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: op_features: 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: flags: 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: create_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.222 INFO:tasks.workunit.client.0.smithi082.stdout: access_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.223 INFO:tasks.workunit.client.0.smithi082.stdout: modify_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.226 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import@snap 2023-10-09T04:13:36.268 INFO:tasks.workunit.client.0.smithi082.stdout:rbd image 'testimg_import': 2023-10-09T04:13:36.268 INFO:tasks.workunit.client.0.smithi082.stdout: size 110 KiB in 1 objects 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: order 22 (4 MiB objects) 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: snapshot_count: 1 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: id: 3a381dd34b2f 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: block_name_prefix: rbd_data.3a381dd34b2f 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: format: 2 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: op_features: 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: flags: 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: create_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: access_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.269 INFO:tasks.workunit.client.0.smithi082.stdout: modify_timestamp: Mon Oct 9 04:13:35 2023 2023-10-09T04:13:36.270 INFO:tasks.workunit.client.0.smithi082.stdout: protected: False 2023-10-09T04:13:36.274 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta list testimg_import 2023-10-09T04:13:36.327 INFO:tasks.workunit.client.0.smithi082.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T04:13:36.327 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 ] 2023-10-09T04:13:36.328 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg_import /tmp/rbd_import_export_65968/img_import 2023-10-09T04:13:36.380 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:13:36.384 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img_import 2023-10-09T04:13:36.384 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/img /tmp/rbd_import_export_65968/img_import 2023-10-09T04:13:36.385 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/img --format %b 2023-10-09T04:13:36.385 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=24 2023-10-09T04:13:36.386 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/img_import --format %b 2023-10-09T04:13:36.386 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=224 2023-10-09T04:13:36.386 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=200 2023-10-09T04:13:36.387 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=200 2023-10-09T04:13:36.387 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T04:13:36.387 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_65968/img_snap 2023-10-09T04:13:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:36 smithi153 bash[27868]: cluster 2023-10-09T04:13:34.902494+0000 mgr.y (mgr.14436) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 202 KiB/s rd, 50 KiB/s wr, 63 op/s 2023-10-09T04:13:36.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:36 smithi153 bash[27868]: cluster 2023-10-09T04:13:35.134850+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:13:36.428 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:13:36.432 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_65968/img_snap_import 2023-10-09T04:13:36.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:36 smithi082 bash[27462]: cluster 2023-10-09T04:13:34.902494+0000 mgr.y (mgr.14436) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 202 KiB/s rd, 50 KiB/s wr, 63 op/s 2023-10-09T04:13:36.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:36 smithi082 bash[27462]: cluster 2023-10-09T04:13:35.134850+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:13:36.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:36 smithi082 bash[33901]: cluster 2023-10-09T04:13:34.902494+0000 mgr.y (mgr.14436) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 202 KiB/s rd, 50 KiB/s wr, 63 op/s 2023-10-09T04:13:36.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:36 smithi082 bash[33901]: cluster 2023-10-09T04:13:35.134850+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:13:36.487 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:13:36.491 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/img_snap /tmp/rbd_import_export_65968/img_snap_import 2023-10-09T04:13:36.491 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/img_snap /tmp/rbd_import_export_65968/img_snap_import 2023-10-09T04:13:36.492 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/img_snap --format %b 2023-10-09T04:13:36.493 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=224 2023-10-09T04:13:36.493 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/img_snap_import --format %b 2023-10-09T04:13:36.493 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=224 2023-10-09T04:13:36.494 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:36.494 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:36.494 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:13:36.494 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:36.494 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_import 2023-10-09T04:13:36.495 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_snap 2023-10-09T04:13:36.495 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_snap_import 2023-10-09T04:13:36.495 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T04:13:37.152 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:37 smithi082 bash[27462]: cluster 2023-10-09T04:13:36.150235+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:13:37.159 INFO:tasks.workunit.client.0.smithi082.stderr: Removing snap: 100% complete...done. 2023-10-09T04:13:37.170 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:13:37.306 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:37.311 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap rm testimg@snap 2023-10-09T04:13:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:37 smithi153 bash[27868]: cluster 2023-10-09T04:13:36.150235+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:13:37.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:37 smithi082 bash[33901]: cluster 2023-10-09T04:13:36.150235+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (3m) 19s ago 3m 13.4M - 0.23.0 ba2b418f427c 96f8d2ca5d45 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (3m) 18s ago 3m 44.5M - 8.3.5 dad864ee21e9 299f8b965a86 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (2m) 19s ago 2m 13.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (2m) 18s ago 2m 11.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443 running (10m) 18s ago 10m 375M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 cd148ccbea9a 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:9283 running (12m) 19s ago 12m 427M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d2bfd06d1f16 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (12m) 19s ago 12m 48.7M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f2d400a9efd 2023-10-09T04:13:38.131 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 running (10m) 18s ago 10m 42.1M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1f2fe6dbb64d 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 running (10m) 19s ago 10m 40.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ae4ed267e219 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (4m) 19s ago 4m 9043k - 1.3.1 1dbe0e931976 75b7372d211c 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (4m) 18s ago 4m 9192k - 1.3.1 1dbe0e931976 1a0055968bab 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (10m) 19s ago 10m 38.3M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9b8a7733f2dc 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (9m) 19s ago 9m 39.0M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 97a8f958f136 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (8m) 19s ago 8m 37.6M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7af0967827d4 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (7m) 19s ago 7m 37.8M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 50ddb6f18074 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (7m) 18s ago 7m 40.7M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (6m) 18s ago 6m 38.0M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (5m) 18s ago 5m 38.1M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (5m) 18s ago 5m 39.6M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:13:38.132 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (2m) 18s ago 4m 53.3M - 2.33.4 514e6a882f6e a6d3dd55e59a 2023-10-09T04:13:38.173 INFO:tasks.workunit.client.0.smithi082.stderr: Removing snap: 100% complete...done. 2023-10-09T04:13:38.183 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:13:38.318 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:38.323 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --order 20 /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:38.403 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:38.404 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:38.407 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:38.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:38 smithi153 bash[27868]: cluster 2023-10-09T04:13:36.903132+0000 mgr.y (mgr.14436) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 67 KiB/s wr, 62 op/s 2023-10-09T04:13:38.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:38 smithi153 bash[27868]: cluster 2023-10-09T04:13:37.154258+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:13:38.461 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:38.466 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65968/img_v2 testimg_import 2023-10-09T04:13:38.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:38 smithi082 bash[33901]: cluster 2023-10-09T04:13:36.903132+0000 mgr.y (mgr.14436) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 67 KiB/s wr, 62 op/s 2023-10-09T04:13:38.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:38 smithi082 bash[33901]: cluster 2023-10-09T04:13:37.154258+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:13:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:38 smithi082 bash[27462]: cluster 2023-10-09T04:13:36.903132+0000 mgr.y (mgr.14436) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 67 KiB/s wr, 62 op/s 2023-10-09T04:13:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:38 smithi082 bash[27462]: cluster 2023-10-09T04:13:37.154258+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:13:38.560 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:13:38.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:13:38.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep order 2023-10-09T04:13:38.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $2} 2023-10-09T04:13:38.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep 20 2023-10-09T04:13:38.617 INFO:tasks.workunit.client.0.smithi082.stdout:20 2023-10-09T04:13:38.617 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:38.726 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:13:38.866 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:38.871 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:13:39.016 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:39.022 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:39.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T04:13:39.086 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:39.090 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg 2023-10-09T04:13:39.090 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep features 2023-10-09T04:13:39.144 INFO:tasks.workunit.client.0.smithi082.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T04:13:39.144 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: 2023-10-09T04:13:39.144 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:39.192 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:39.196 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65968/img_v2 testimg_import 2023-10-09T04:13:39.274 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:13:39.279 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info+ testimg_import 2023-10-09T04:13:39.279 INFO:tasks.workunit.client.0.smithi082.stderr:grep features 2023-10-09T04:13:39.305 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:39 smithi082 bash[33901]: audit 2023-10-09T04:13:38.108554+0000 mgr.y (mgr.14436) 174 : audit [DBG] from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:39.305 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:39 smithi082 bash[33901]: cluster 2023-10-09T04:13:38.167005+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:13:39.305 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:39 smithi082 bash[27462]: audit 2023-10-09T04:13:38.108554+0000 mgr.y (mgr.14436) 174 : audit [DBG] from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:39.305 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:39 smithi082 bash[27462]: cluster 2023-10-09T04:13:38.167005+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:13:39.332 INFO:tasks.workunit.client.0.smithi082.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T04:13:39.332 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: 2023-10-09T04:13:39.332 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ features: layering 2023-10-09T04:13:39.332 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: != features: layering 2023-10-09T04:13:39.333 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: ] 2023-10-09T04:13:39.333 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:39.333 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:13:39.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:39 smithi153 bash[27868]: audit 2023-10-09T04:13:38.108554+0000 mgr.y (mgr.14436) 174 : audit [DBG] from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:13:39.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:39 smithi153 bash[27868]: cluster 2023-10-09T04:13:38.167005+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:13:39.442 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:39.446 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:13:39.544 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:39.548 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:39.640 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:39.644 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:39.699 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:39.703 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65968/img_v2 testimg_import 2023-10-09T04:13:39.802 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:13:39.810 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:13:39.810 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep stripe unit 2023-10-09T04:13:39.810 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T04:13:39.864 INFO:tasks.workunit.client.0.smithi082.stdout: stripe unit: 4 KiB 2023-10-09T04:13:39.865 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:13:39.865 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep stripe count 2023-10-09T04:13:39.865 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $3} 2023-10-09T04:13:39.865 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep 1000 2023-10-09T04:13:39.918 INFO:tasks.workunit.client.0.smithi082.stdout:1000 2023-10-09T04:13:39.918 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/img_v2 2023-10-09T04:13:39.919 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:13:40.068 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:13:40.072 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:13:40.212 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:13:40.219 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_65968/img testimg 2023-10-09T04:13:40.300 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:40.304 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap1 2023-10-09T04:13:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:40 smithi153 bash[27868]: cluster 2023-10-09T04:13:38.903585+0000 mgr.y (mgr.14436) 175 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2023-10-09T04:13:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:40 smithi082 bash[27462]: cluster 2023-10-09T04:13:38.903585+0000 mgr.y (mgr.14436) 175 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2023-10-09T04:13:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:40 smithi082 bash[33901]: cluster 2023-10-09T04:13:38.903585+0000 mgr.y (mgr.14436) 175 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2023-10-09T04:13:41.196 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:13:41.207 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap2 2023-10-09T04:13:42.213 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:13:42.224 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap protect testimg@snap2 2023-10-09T04:13:42.224 INFO:teuthology.orchestra.run.smithi082.stdout:wait for minority of mons to upgrade 2023-10-09T04:13:42.278 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_65968/snap_protect 2023-10-09T04:13:42.332 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:42.338 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_65968/snap_protect testimg_import 2023-10-09T04:13:42.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:42 smithi082 bash[33901]: cluster 2023-10-09T04:13:40.904013+0000 mgr.y (mgr.14436) 176 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 151 op/s 2023-10-09T04:13:42.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:42 smithi082 bash[33901]: cluster 2023-10-09T04:13:41.183676+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:13:42.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:42 smithi082 bash[27462]: cluster 2023-10-09T04:13:40.904013+0000 mgr.y (mgr.14436) 176 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 151 op/s 2023-10-09T04:13:42.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:42 smithi082 bash[27462]: cluster 2023-10-09T04:13:41.183676+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:13:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:42 smithi153 bash[27868]: cluster 2023-10-09T04:13:40.904013+0000 mgr.y (mgr.14436) 176 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 151 op/s 2023-10-09T04:13:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:42 smithi153 bash[27868]: cluster 2023-10-09T04:13:41.183676+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:13:43.176 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:42 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:42] "GET /metrics HTTP/1.1" 200 191881 "" "Prometheus/2.33.4" 2023-10-09T04:13:43.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T04:13:43.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:43 smithi082 bash[27462]: cluster 2023-10-09T04:13:42.194233+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:13:43.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:43 smithi082 bash[33901]: cluster 2023-10-09T04:13:42.194233+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:13:43.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:43 smithi153 bash[27868]: cluster 2023-10-09T04:13:42.194233+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:13:44.126 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:13:44.130 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbd info testimg_import@snap1 2023-10-09T04:13:44.130 INFO:tasks.workunit.client.0.smithi082.stderr:grep protected: False 2023-10-09T04:13:44.175 INFO:tasks.workunit.client.0.smithi082.stdout: protected: False 2023-10-09T04:13:44.176 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbd info testimg_import@snap2 2023-10-09T04:13:44.176 INFO:tasks.workunit.client.0.smithi082.stderr:grep protected: True 2023-10-09T04:13:44.228 INFO:tasks.workunit.client.0.smithi082.stdout: protected: True 2023-10-09T04:13:44.228 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/snap_protect 2023-10-09T04:13:44.229 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T04:13:44.284 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T04:13:44.336 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap purge testimg 2023-10-09T04:13:44.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:43 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:44 smithi153 bash[27868]: cluster 2023-10-09T04:13:42.904827+0000 mgr.y (mgr.14436) 177 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 44 KiB/s wr, 504 op/s 2023-10-09T04:13:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:44 smithi153 bash[27868]: cluster 2023-10-09T04:13:43.202103+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:13:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:44 smithi153 bash[27868]: cluster 2023-10-09T04:13:44.095805+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:13:44.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[33901]: cluster 2023-10-09T04:13:42.904827+0000 mgr.y (mgr.14436) 177 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 44 KiB/s wr, 504 op/s 2023-10-09T04:13:44.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[33901]: cluster 2023-10-09T04:13:43.202103+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:13:44.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[33901]: cluster 2023-10-09T04:13:44.095805+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:13:44.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[27462]: cluster 2023-10-09T04:13:42.904827+0000 mgr.y (mgr.14436) 177 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 44 KiB/s wr, 504 op/s 2023-10-09T04:13:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[27462]: cluster 2023-10-09T04:13:43.202103+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:13:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:44 smithi082 bash[27462]: cluster 2023-10-09T04:13:44.095805+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:13:46.223 INFO:tasks.workunit.client.0.smithi082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:13:46.236 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap purge testimg_import 2023-10-09T04:13:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:46 smithi082 bash[27462]: cluster 2023-10-09T04:13:44.905332+0000 mgr.y (mgr.14436) 178 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 378 KiB/s rd, 48 KiB/s wr, 507 op/s 2023-10-09T04:13:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:46 smithi082 bash[27462]: cluster 2023-10-09T04:13:45.210291+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:13:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:46 smithi082 bash[33901]: cluster 2023-10-09T04:13:44.905332+0000 mgr.y (mgr.14436) 178 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 378 KiB/s rd, 48 KiB/s wr, 507 op/s 2023-10-09T04:13:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:46 smithi082 bash[33901]: cluster 2023-10-09T04:13:45.210291+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:13:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:46 smithi153 bash[27868]: cluster 2023-10-09T04:13:44.905332+0000 mgr.y (mgr.14436) 178 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 378 KiB/s rd, 48 KiB/s wr, 507 op/s 2023-10-09T04:13:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:46 smithi153 bash[27868]: cluster 2023-10-09T04:13:45.210291+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:13:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:47 smithi082 bash[27462]: cluster 2023-10-09T04:13:46.213818+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:13:47.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:47 smithi082 bash[33901]: cluster 2023-10-09T04:13:46.213818+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:13:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:47 smithi153 bash[27868]: cluster 2023-10-09T04:13:46.213818+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:13:48.247 INFO:tasks.workunit.client.0.smithi082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:13:48.256 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:13:48.390 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:48.393 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:13:48.510 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:48.515 INFO:tasks.workunit.client.0.smithi082.stderr:+ tiered=0 2023-10-09T04:13:48.515 INFO:tasks.workunit.client.0.smithi082.stderr:+ ceph osd dump+ 2023-10-09T04:13:48.515 INFO:tasks.workunit.client.0.smithi082.stderr:grep ^pool 2023-10-09T04:13:48.515 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep 'rbd' 2023-10-09T04:13:48.515 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep tier 2023-10-09T04:13:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:48 smithi153 bash[27868]: cluster 2023-10-09T04:13:46.905799+0000 mgr.y (mgr.14436) 179 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:48 smithi153 bash[27868]: cluster 2023-10-09T04:13:47.228847+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:13:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:48 smithi082 bash[27462]: cluster 2023-10-09T04:13:46.905799+0000 mgr.y (mgr.14436) 179 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:48 smithi082 bash[27462]: cluster 2023-10-09T04:13:47.228847+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:13:48.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:48 smithi082 bash[33901]: cluster 2023-10-09T04:13:46.905799+0000 mgr.y (mgr.14436) 179 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:48 smithi082 bash[33901]: cluster 2023-10-09T04:13:47.228847+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:13:48.933 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_65968/sparse1 2023-10-09T04:13:48.939 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:13:48.939 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:13:48.939 INFO:tasks.workunit.client.0.smithi082.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00527091 s, 199 MB/s 2023-10-09T04:13:48.939 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_65968/sparse2 2023-10-09T04:13:48.943 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:13:48.943 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:13:48.944 INFO:tasks.workunit.client.0.smithi082.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00394855 s, 266 MB/s 2023-10-09T04:13:48.944 INFO:tasks.workunit.client.0.smithi082.stderr:+ truncate /tmp/rbd_import_export_65968/sparse2 -s 2M 2023-10-09T04:13:49.131 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:13:49.183 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:13:49.183 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:13:49.188 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:49.188 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65968/sparse1 2023-10-09T04:13:49.277 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:13:49.278 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:49.278 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:49.282 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:13:49.282 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep sparse1 2023-10-09T04:13:49.282 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T04:13:49.329 INFO:tasks.workunit.client.0.smithi082.stdout:sparse1 2 MiB 2 2023-10-09T04:13:49.330 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse1 2023-10-09T04:13:49.330 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:13:49.330 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:13:49.330 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:49.330 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:49.375 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.3b58ef8ffced 2023-10-09T04:13:49.375 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep rbd_data.3b58ef8ffced 2023-10-09T04:13:49.375 INFO:tasks.workunit.client.0.smithi082.stderr:get_image_data_pool sparse1 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.3b58ef8ffced\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbd info sparse1 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr:grep+ awk -F: { print $NF } 2023-10-09T04:13:49.376 INFO:tasks.workunit.client.0.smithi082.stderr: data_pool: 2023-10-09T04:13:49.420 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:49.420 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:49.420 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:49.421 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:49.421 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:49.465 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:13:49.465 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:13:49.465 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:13:49.465 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:49.514 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:49.519 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:49.519 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:49.521 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse1 --format %b 2023-10-09T04:13:49.522 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:13:49.523 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse1.out --format %b 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:49.524 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:13:49.667 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:13:49.671 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:13:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:49 smithi153 bash[27868]: cluster 2023-10-09T04:13:48.233973+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:49 smithi153 bash[27868]: audit 2023-10-09T04:13:48.808500+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:49 smithi153 bash[27868]: audit 2023-10-09T04:13:48.918409+0000 mon.a (mon.0) 758 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.724 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:13:49.724 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:13:49.729 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:49.730 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65968/sparse2 2023-10-09T04:13:49.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[27462]: cluster 2023-10-09T04:13:48.233973+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:49.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[27462]: audit 2023-10-09T04:13:48.808500+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[27462]: audit 2023-10-09T04:13:48.918409+0000 mon.a (mon.0) 758 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[33901]: cluster 2023-10-09T04:13:48.233973+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:49.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[33901]: audit 2023-10-09T04:13:48.808500+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:49 smithi082 bash[33901]: audit 2023-10-09T04:13:48.918409+0000 mon.a (mon.0) 758 : audit [DBG] from='client.? 172.21.15.82:0/3290032016' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:13:49.820 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:13:49.821 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:49.821 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:49.825 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:13:49.825 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grepgrep sparse2 -Ei 2023-10-09T04:13:49.825 INFO:tasks.workunit.client.0.smithi082.stderr: (2 MiB|2048k) 2023-10-09T04:13:49.876 INFO:tasks.workunit.client.0.smithi082.stdout:sparse2 2 MiB 2 2023-10-09T04:13:49.878 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse2 2023-10-09T04:13:49.878 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:13:49.879 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:13:49.879 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:49.879 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:49.930 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.623ca67bac5e 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.623ca67bac5e 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse2 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.623ca67bac5e\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:49.931 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:13:49.932 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:13:49.932 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:13:49.981 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:49.981 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:49.981 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:49.981 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:49.981 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:50.029 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:13:50.030 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2023-10-09T04:13:50.030 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T04:13:50.030 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:50.079 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:50.084 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:50.084 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:50.087 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse2 --format %b 2023-10-09T04:13:50.087 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:13:50.088 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse2.out --format %b 2023-10-09T04:13:50.088 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:13:50.088 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:50.089 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:50.089 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:13:50.089 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:50.089 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:13:50.220 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:13:50.225 INFO:tasks.workunit.client.0.smithi082.stderr:+ truncate /tmp/rbd_import_export_65968/sparse1 -s 10M 2023-10-09T04:13:50.226 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T04:13:50.370 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:50.371 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:50.371 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:50.375 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:13:50.375 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep sparse1 2023-10-09T04:13:50.375 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T04:13:50.429 INFO:tasks.workunit.client.0.smithi082.stdout:sparse1 10 MiB 2 2023-10-09T04:13:50.429 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse1 2023-10-09T04:13:50.429 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:13:50.430 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:13:50.430 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:50.430 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.6269c849e21a 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.6269c849e21a 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse1 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:+ + image=sparse1 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:sed -e s/rbd_data.6269c849e21a\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:50.486 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:50.487 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:13:50.487 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:13:50.487 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:13:50.539 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:50.540 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:50.540 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:50.540 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:50.540 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:50.581 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:13:50.581 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:13:50.581 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:13:50.581 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:50.633 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:50.638 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:50.638 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/sparse1 /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:50.650 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse1 --format %b 2023-10-09T04:13:50.651 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:13:50.652 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse1.out --format %b 2023-10-09T04:13:50.652 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:13:50.653 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:50.653 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:50.653 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:13:50.653 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse1.out 2023-10-09T04:13:50.654 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:50 smithi082 bash[27462]: cluster 2023-10-09T04:13:48.906279+0000 mgr.y (mgr.14436) 180 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-10-09T04:13:50.655 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:50 smithi082 bash[33901]: cluster 2023-10-09T04:13:48.906279+0000 mgr.y (mgr.14436) 180 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-10-09T04:13:50.655 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:13:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:50 smithi153 bash[27868]: cluster 2023-10-09T04:13:48.906279+0000 mgr.y (mgr.14436) 180 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-10-09T04:13:50.802 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:13:50.807 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_65968/sparse2 oflag=append conv=notrunc 2023-10-09T04:13:50.819 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:13:50.819 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:13:50.819 INFO:tasks.workunit.client.0.smithi082.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0102011 s, 206 MB/s 2023-10-09T04:13:50.819 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T04:13:50.947 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:50.948 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:50.948 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:50.952 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:13:50.952 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep sparse2 2023-10-09T04:13:50.952 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T04:13:51.000 INFO:tasks.workunit.client.0.smithi082.stdout:sparse2 4 MiB 2 2023-10-09T04:13:51.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse2 2023-10-09T04:13:51.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:13:51.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:13:51.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:51.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:51.053 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.6296f537ec54 2023-10-09T04:13:51.053 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep rbd_data.6296f537ec54 2023-10-09T04:13:51.053 INFO:tasks.workunit.client.0.smithi082.stderr:get_image_data_pool sparse2 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.6296f537ec54\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:13:51.054 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:13:51.097 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:51.097 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:51.097 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:51.097 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:51.097 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:2 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:3 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2 3 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T04:13:51.140 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:51.194 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T04:13:51.199 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_65968/sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:51.199 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_65968/sparse2 /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:51.204 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse2 --format %b 2023-10-09T04:13:51.205 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=6144 2023-10-09T04:13:51.205 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse2.out --format %b 2023-10-09T04:13:51.206 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=6144 2023-10-09T04:13:51.206 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:51.206 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:13:51.206 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:13:51.207 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse2.out 2023-10-09T04:13:51.207 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:13:51.352 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T04:13:51.357 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T04:13:51.357 INFO:tasks.workunit.client.0.smithi082.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T04:13:51.357 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_65968/sparse1 sparse 2023-10-09T04:13:51.449 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T04:13:51.451 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:51.451 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:13:51.454 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:13:51.454 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:51.455 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:51.455 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:51.455 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:51.504 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.3c154ebc609f 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.3c154ebc609f 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.3c154ebc609f\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:51.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:51.507 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:13:51.507 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:13:51.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:51.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:51.566 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:51.566 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:51.566 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:51.610 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:13:51.610 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:13:51.610 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:13:51.610 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:13:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:51 smithi153 bash[27868]: cluster 2023-10-09T04:13:50.906940+0000 mgr.y (mgr.14436) 181 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 5.4 KiB/s wr, 94 op/s 2023-10-09T04:13:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:51 smithi153 bash[27868]: audit 2023-10-09T04:13:51.246725+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.82:0/3927518236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:51.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:51 smithi082 bash[27462]: cluster 2023-10-09T04:13:50.906940+0000 mgr.y (mgr.14436) 181 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 5.4 KiB/s wr, 94 op/s 2023-10-09T04:13:51.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:51 smithi082 bash[27462]: audit 2023-10-09T04:13:51.246725+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.82:0/3927518236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:51.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:51 smithi082 bash[33901]: cluster 2023-10-09T04:13:50.906940+0000 mgr.y (mgr.14436) 181 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 5.4 KiB/s wr, 94 op/s 2023-10-09T04:13:51.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:51 smithi082 bash[33901]: audit 2023-10-09T04:13:51.246725+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.82:0/3927518236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:51.750 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:13:51.754 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T04:13:51.754 INFO:tasks.workunit.client.0.smithi082.stdout:zeros import through stdin to sparse image 2023-10-09T04:13:51.755 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T04:13:51.755 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format - sparse 2023-10-09T04:13:51.826 INFO:tasks.workunit.client.0.smithi082.stderr:4+0 records in 2023-10-09T04:13:51.826 INFO:tasks.workunit.client.0.smithi082.stderr:4+0 records out 2023-10-09T04:13:51.826 INFO:tasks.workunit.client.0.smithi082.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0706578 s, 59.4 MB/s 2023-10-09T04:13:51.844 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:13:51.845 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:51.850 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:13:51.851 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:51.851 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:51.851 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:51.851 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:51.909 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.3c3663ef278c 2023-10-09T04:13:51.909 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.3c3663ef278c 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed+ sort -u 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr: -e+ rbd info sparse 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr: s/rbd_data.3c3663ef278c\.//+ grep data_pool: 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr: -e+ awk -F: { print $NF } 2023-10-09T04:13:51.910 INFO:tasks.workunit.client.0.smithi082.stderr: s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:51.962 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:51.962 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:51.962 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:51.962 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:51.962 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:52.010 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects= 2023-10-09T04:13:52.010 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 2023-10-09T04:13:52.010 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T04:13:52.010 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:13:52.147 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:52.151 INFO:tasks.workunit.client.0.smithi082.stdout:zeros export to sparse file 2023-10-09T04:13:52.151 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo zeros export to sparse file 2023-10-09T04:13:52.151 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T04:13:52.205 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:13:52.209 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:52.209 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:52.209 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.3c573850417c 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:13:52.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:13:52.324 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:52.325 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:52.325 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:52.325 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:52.325 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados -p rbd put rbd_data.3c573850417c.000000000000 - 2023-10-09T04:13:52.363 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:13:52.363 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:13:52.363 INFO:tasks.workunit.client.0.smithi082.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0909808 s, 46.1 MB/s 2023-10-09T04:13:52.389 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:13:52.389 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:52.389 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:52.390 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:13:52.390 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:13:52.447 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.3c573850417c 2023-10-09T04:13:52.447 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.3c573850417c 2023-10-09T04:13:52.447 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:13:52.447 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:13:52.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.3c573850417c\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:13:52.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:13:52.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:13:52.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ + awk -F: { print $NF } 2023-10-09T04:13:52.449 INFO:tasks.workunit.client.0.smithi082.stderr:grep data_pool: 2023-10-09T04:13:52.495 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:13:52.496 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:13:52.496 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:13:52.496 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:13:52.496 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:13:52.543 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:13:52.543 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2023-10-09T04:13:52.543 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T04:13:52.543 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse 2023-10-09T04:13:52.544 INFO:tasks.workunit.client.0.smithi082.stderr:rm: cannot remove '/tmp/rbd_import_export_65968/sparse': No such file or directory 2023-10-09T04:13:52.544 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:52.544 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse /tmp/rbd_import_export_65968/sparse 2023-10-09T04:13:52.592 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:13:52.597 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_65968/sparse --format=%b 2023-10-09T04:13:52.598 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 = 0 ] 2023-10-09T04:13:52.598 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:13:52.734 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:13:52.738 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_65968/sparse /tmp/rbd_import_export_65968/sparse1 /tmp/rbd_import_export_65968/sparse2 /tmp/rbd_import_export_65968/sparse3 2023-10-09T04:13:52.742 INFO:tasks.workunit.client.0.smithi082.stderr:rm: cannot remove '/tmp/rbd_import_export_65968/sparse3': No such file or directory 2023-10-09T04:13:52.742 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:13:52.743 INFO:tasks.workunit.client.0.smithi082.stdout:OK 2023-10-09T04:13:52.744 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo OK 2023-10-09T04:13:52.744 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm -rf /tmp/rbd_import_export_65968 2023-10-09T04:13:52.744 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T04:13:52.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T04:13:52.755 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T04:13:52.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:13:53.049 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:13:52 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:52] "GET /metrics HTTP/1.1" 200 192041 "" "Prometheus/2.33.4" 2023-10-09T04:13:53.050 DEBUG:teuthology.parallel:result is None 2023-10-09T04:13:53.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:13:53.062 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:13:53.062 DEBUG:teuthology.orchestra.run.smithi082:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:13:53.111 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:13:53.111 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:13:53.111 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T04:13:53.111 DEBUG:teuthology.parallel:result is None 2023-10-09T04:13:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:53 smithi082 bash[27462]: cluster 2023-10-09T04:13:52.907624+0000 mgr.y (mgr.14436) 182 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 467 op/s 2023-10-09T04:13:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:53 smithi082 bash[27462]: audit 2023-10-09T04:13:53.737740+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.82:0/3259160344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:53 smithi082 bash[33901]: cluster 2023-10-09T04:13:52.907624+0000 mgr.y (mgr.14436) 182 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 467 op/s 2023-10-09T04:13:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:53 smithi082 bash[33901]: audit 2023-10-09T04:13:53.737740+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.82:0/3259160344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:54.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:13:53 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:13:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:53 smithi153 bash[27868]: cluster 2023-10-09T04:13:52.907624+0000 mgr.y (mgr.14436) 182 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 467 op/s 2023-10-09T04:13:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:53 smithi153 bash[27868]: audit 2023-10-09T04:13:53.737740+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.82:0/3259160344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:55.411 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:55 smithi082 bash[33901]: cluster 2023-10-09T04:13:54.102753+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:55.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:55 smithi082 bash[27462]: cluster 2023-10-09T04:13:54.102753+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:55 smithi153 bash[27868]: cluster 2023-10-09T04:13:54.102753+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:56 smithi153 bash[27868]: cluster 2023-10-09T04:13:54.908257+0000 mgr.y (mgr.14436) 183 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 407 op/s 2023-10-09T04:13:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:56 smithi082 bash[27462]: cluster 2023-10-09T04:13:54.908257+0000 mgr.y (mgr.14436) 183 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 407 op/s 2023-10-09T04:13:56.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:56 smithi082 bash[33901]: cluster 2023-10-09T04:13:54.908257+0000 mgr.y (mgr.14436) 183 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 407 op/s 2023-10-09T04:13:57.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:57 smithi153 bash[27868]: audit 2023-10-09T04:13:56.144189+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.82:0/2502760435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:57.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:57 smithi082 bash[27462]: audit 2023-10-09T04:13:56.144189+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.82:0/2502760435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:57.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:57 smithi082 bash[33901]: audit 2023-10-09T04:13:56.144189+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.82:0/2502760435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:58.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:58 smithi153 bash[27868]: cluster 2023-10-09T04:13:56.908665+0000 mgr.y (mgr.14436) 184 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 932 KiB/s rd, 1.3 MiB/s wr, 298 op/s 2023-10-09T04:13:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:58 smithi082 bash[27462]: cluster 2023-10-09T04:13:56.908665+0000 mgr.y (mgr.14436) 184 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 932 KiB/s rd, 1.3 MiB/s wr, 298 op/s 2023-10-09T04:13:58.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:58 smithi082 bash[33901]: cluster 2023-10-09T04:13:56.908665+0000 mgr.y (mgr.14436) 184 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 932 KiB/s rd, 1.3 MiB/s wr, 298 op/s 2023-10-09T04:13:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:13:59 smithi153 bash[27868]: audit 2023-10-09T04:13:58.554517+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.82:0/1195181971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:59.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:13:59 smithi082 bash[33901]: audit 2023-10-09T04:13:58.554517+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.82:0/1195181971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:59.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:13:59 smithi082 bash[27462]: audit 2023-10-09T04:13:58.554517+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.82:0/1195181971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:00.398 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:00 smithi082 bash[27462]: cluster 2023-10-09T04:13:58.909384+0000 mgr.y (mgr.14436) 185 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:00.400 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:00 smithi082 bash[33901]: cluster 2023-10-09T04:13:58.909384+0000 mgr.y (mgr.14436) 185 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:00 smithi153 bash[27868]: cluster 2023-10-09T04:13:58.909384+0000 mgr.y (mgr.14436) 185 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:01 smithi153 bash[27868]: audit 2023-10-09T04:14:00.962238+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.82:0/3475549524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:01 smithi082 bash[27462]: audit 2023-10-09T04:14:00.962238+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.82:0/3475549524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:01.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:01 smithi082 bash[33901]: audit 2023-10-09T04:14:00.962238+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.82:0/3475549524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:02 smithi153 bash[27868]: cluster 2023-10-09T04:14:00.909684+0000 mgr.y (mgr.14436) 186 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:02 smithi153 bash[27868]: audit 2023-10-09T04:14:01.417042+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:02 smithi153 bash[27868]: audit 2023-10-09T04:14:01.419339+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:02 smithi153 bash[27868]: audit 2023-10-09T04:14:01.426784+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:02.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[27462]: cluster 2023-10-09T04:14:00.909684+0000 mgr.y (mgr.14436) 186 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:02.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[27462]: audit 2023-10-09T04:14:01.417042+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[27462]: audit 2023-10-09T04:14:01.419339+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[27462]: audit 2023-10-09T04:14:01.426784+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[33901]: cluster 2023-10-09T04:14:00.909684+0000 mgr.y (mgr.14436) 186 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[33901]: audit 2023-10-09T04:14:01.417042+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[33901]: audit 2023-10-09T04:14:01.419339+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[33901]: audit 2023-10-09T04:14:01.426784+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:03.162 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:02] "GET /metrics HTTP/1.1" 200 192041 "" "Prometheus/2.33.4" 2023-10-09T04:14:03.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:03 smithi153 bash[27868]: cephadm 2023-10-09T04:14:01.417664+0000 mgr.y (mgr.14436) 187 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:14:03.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:03 smithi153 bash[27868]: cephadm 2023-10-09T04:14:01.417883+0000 mgr.y (mgr.14436) 188 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:14:03.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:03 smithi153 bash[27868]: cephadm 2023-10-09T04:14:01.427902+0000 mgr.y (mgr.14436) 189 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:14:03.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:03 smithi153 bash[27868]: cephadm 2023-10-09T04:14:01.651286+0000 mgr.y (mgr.14436) 190 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi153 2023-10-09T04:14:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[27462]: cephadm 2023-10-09T04:14:01.417664+0000 mgr.y (mgr.14436) 187 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:14:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[27462]: cephadm 2023-10-09T04:14:01.417883+0000 mgr.y (mgr.14436) 188 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[27462]: cephadm 2023-10-09T04:14:01.427902+0000 mgr.y (mgr.14436) 189 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[27462]: cephadm 2023-10-09T04:14:01.651286+0000 mgr.y (mgr.14436) 190 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi153 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[33901]: cephadm 2023-10-09T04:14:01.417664+0000 mgr.y (mgr.14436) 187 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[33901]: cephadm 2023-10-09T04:14:01.417883+0000 mgr.y (mgr.14436) 188 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[33901]: cephadm 2023-10-09T04:14:01.427902+0000 mgr.y (mgr.14436) 189 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:14:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:03 smithi082 bash[33901]: cephadm 2023-10-09T04:14:01.651286+0000 mgr.y (mgr.14436) 190 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi153 2023-10-09T04:14:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:04 smithi153 bash[27868]: cluster 2023-10-09T04:14:02.910667+0000 mgr.y (mgr.14436) 191 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:04 smithi153 bash[27868]: audit 2023-10-09T04:14:03.367304+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.82:0/3714766270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:04.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:03 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:14:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:04 smithi082 bash[33901]: cluster 2023-10-09T04:14:02.910667+0000 mgr.y (mgr.14436) 191 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:04 smithi082 bash[33901]: audit 2023-10-09T04:14:03.367304+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.82:0/3714766270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:04.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:04 smithi082 bash[27462]: cluster 2023-10-09T04:14:02.910667+0000 mgr.y (mgr.14436) 191 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:04.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:04 smithi082 bash[27462]: audit 2023-10-09T04:14:03.367304+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.82:0/3714766270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:06 smithi153 bash[27868]: cluster 2023-10-09T04:14:04.911264+0000 mgr.y (mgr.14436) 192 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:06 smithi153 bash[27868]: audit 2023-10-09T04:14:05.771366+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.82:0/4216851146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:06 smithi082 bash[33901]: cluster 2023-10-09T04:14:04.911264+0000 mgr.y (mgr.14436) 192 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:06 smithi082 bash[33901]: audit 2023-10-09T04:14:05.771366+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.82:0/4216851146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:06 smithi082 bash[27462]: cluster 2023-10-09T04:14:04.911264+0000 mgr.y (mgr.14436) 192 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:06 smithi082 bash[27462]: audit 2023-10-09T04:14:05.771366+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.82:0/4216851146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:08.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:08 smithi153 bash[27868]: cluster 2023-10-09T04:14:06.911944+0000 mgr.y (mgr.14436) 193 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:08.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:08 smithi082 bash[27462]: cluster 2023-10-09T04:14:06.911944+0000 mgr.y (mgr.14436) 193 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:08.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:08 smithi082 bash[33901]: cluster 2023-10-09T04:14:06.911944+0000 mgr.y (mgr.14436) 193 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:09.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:09 smithi082 bash[27462]: audit 2023-10-09T04:14:08.189785+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.82:0/2266905054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:09.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:09 smithi082 bash[33901]: audit 2023-10-09T04:14:08.189785+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.82:0/2266905054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:09 smithi153 bash[27868]: audit 2023-10-09T04:14:08.189785+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.82:0/2266905054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:10.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:10 smithi082 bash[27462]: cluster 2023-10-09T04:14:08.913096+0000 mgr.y (mgr.14436) 194 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:10.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:10 smithi082 bash[33901]: cluster 2023-10-09T04:14:08.913096+0000 mgr.y (mgr.14436) 194 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:10.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:10 smithi153 bash[27868]: cluster 2023-10-09T04:14:08.913096+0000 mgr.y (mgr.14436) 194 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:11.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:11 smithi082 bash[27462]: audit 2023-10-09T04:14:10.595559+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.82:0/2338929158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:11.200 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:11 smithi082 bash[33901]: audit 2023-10-09T04:14:10.595559+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.82:0/2338929158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:11.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:11 smithi153 bash[27868]: audit 2023-10-09T04:14:10.595559+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.82:0/2338929158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:12.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:12 smithi082 bash[27462]: cluster 2023-10-09T04:14:10.913791+0000 mgr.y (mgr.14436) 195 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:12.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:12 smithi082 bash[33901]: cluster 2023-10-09T04:14:10.913791+0000 mgr.y (mgr.14436) 195 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:12 smithi153 bash[27868]: cluster 2023-10-09T04:14:10.913791+0000 mgr.y (mgr.14436) 195 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:13.211 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:12] "GET /metrics HTTP/1.1" 200 192087 "" "Prometheus/2.33.4" 2023-10-09T04:14:13.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:13 smithi082 bash[27462]: audit 2023-10-09T04:14:13.011624+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.82:0/1632337640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:13.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:13 smithi082 bash[33901]: audit 2023-10-09T04:14:13.011624+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.82:0/1632337640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:13.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:13 smithi153 bash[27868]: audit 2023-10-09T04:14:13.011624+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.82:0/1632337640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:14 smithi153 bash[27868]: cluster 2023-10-09T04:14:12.914861+0000 mgr.y (mgr.14436) 196 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:14.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:13 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:14:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:14 smithi082 bash[27462]: cluster 2023-10-09T04:14:12.914861+0000 mgr.y (mgr.14436) 196 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:14 smithi082 bash[33901]: cluster 2023-10-09T04:14:12.914861+0000 mgr.y (mgr.14436) 196 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:16 smithi082 bash[27462]: cluster 2023-10-09T04:14:14.915523+0000 mgr.y (mgr.14436) 197 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:16 smithi082 bash[27462]: audit 2023-10-09T04:14:15.422679+0000 mon.a (mon.0) 769 : audit [DBG] from='client.? 172.21.15.82:0/3941546210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:16 smithi082 bash[33901]: cluster 2023-10-09T04:14:14.915523+0000 mgr.y (mgr.14436) 197 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:16 smithi082 bash[33901]: audit 2023-10-09T04:14:15.422679+0000 mon.a (mon.0) 769 : audit [DBG] from='client.? 172.21.15.82:0/3941546210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:16 smithi153 bash[27868]: cluster 2023-10-09T04:14:14.915523+0000 mgr.y (mgr.14436) 197 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:16 smithi153 bash[27868]: audit 2023-10-09T04:14:15.422679+0000 mon.a (mon.0) 769 : audit [DBG] from='client.? 172.21.15.82:0/3941546210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:18.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:18 smithi082 bash[27462]: cluster 2023-10-09T04:14:16.916110+0000 mgr.y (mgr.14436) 198 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:18.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:18 smithi082 bash[27462]: audit 2023-10-09T04:14:17.831872+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.82:0/3658929275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:18.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:18 smithi082 bash[33901]: cluster 2023-10-09T04:14:16.916110+0000 mgr.y (mgr.14436) 198 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:18.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:18 smithi082 bash[33901]: audit 2023-10-09T04:14:17.831872+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.82:0/3658929275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:18 smithi153 bash[27868]: cluster 2023-10-09T04:14:16.916110+0000 mgr.y (mgr.14436) 198 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:18 smithi153 bash[27868]: audit 2023-10-09T04:14:17.831872+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.82:0/3658929275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:20 smithi082 bash[27462]: cluster 2023-10-09T04:14:18.917279+0000 mgr.y (mgr.14436) 199 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:20 smithi082 bash[33901]: cluster 2023-10-09T04:14:18.917279+0000 mgr.y (mgr.14436) 199 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:20 smithi153 bash[27868]: cluster 2023-10-09T04:14:18.917279+0000 mgr.y (mgr.14436) 199 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:21.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:21 smithi082 bash[33901]: audit 2023-10-09T04:14:20.238334+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.82:0/2673304898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:21 smithi082 bash[27462]: audit 2023-10-09T04:14:20.238334+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.82:0/2673304898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:21.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:21 smithi153 bash[27868]: audit 2023-10-09T04:14:20.238334+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.82:0/2673304898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:22 smithi153 bash[27868]: cluster 2023-10-09T04:14:20.917871+0000 mgr.y (mgr.14436) 200 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:22 smithi082 bash[33901]: cluster 2023-10-09T04:14:20.917871+0000 mgr.y (mgr.14436) 200 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:22.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:22 smithi082 bash[27462]: cluster 2023-10-09T04:14:20.917871+0000 mgr.y (mgr.14436) 200 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:23.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:22] "GET /metrics HTTP/1.1" 200 192087 "" "Prometheus/2.33.4" 2023-10-09T04:14:23.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:23 smithi153 bash[27868]: audit 2023-10-09T04:14:22.648107+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.82:0/4288417479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:23.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:23 smithi082 bash[27462]: audit 2023-10-09T04:14:22.648107+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.82:0/4288417479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:23.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:23 smithi082 bash[33901]: audit 2023-10-09T04:14:22.648107+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.82:0/4288417479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:24.248 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:23 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:14:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:24 smithi153 bash[27868]: cluster 2023-10-09T04:14:22.918953+0000 mgr.y (mgr.14436) 201 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:24.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:24 smithi082 bash[27462]: cluster 2023-10-09T04:14:22.918953+0000 mgr.y (mgr.14436) 201 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:24.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:24 smithi082 bash[33901]: cluster 2023-10-09T04:14:22.918953+0000 mgr.y (mgr.14436) 201 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:25 smithi153 bash[27868]: audit 2023-10-09T04:14:25.051482+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.82:0/2189636787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:25.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:25 smithi082 bash[27462]: audit 2023-10-09T04:14:25.051482+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.82:0/2189636787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:25.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:25 smithi082 bash[33901]: audit 2023-10-09T04:14:25.051482+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.82:0/2189636787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:26 smithi153 bash[27868]: cluster 2023-10-09T04:14:24.919625+0000 mgr.y (mgr.14436) 202 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:26.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:26 smithi082 bash[27462]: cluster 2023-10-09T04:14:24.919625+0000 mgr.y (mgr.14436) 202 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:26.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:26 smithi082 bash[33901]: cluster 2023-10-09T04:14:24.919625+0000 mgr.y (mgr.14436) 202 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:28 smithi153 bash[27868]: cluster 2023-10-09T04:14:26.920400+0000 mgr.y (mgr.14436) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:28 smithi153 bash[27868]: audit 2023-10-09T04:14:27.462177+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.82:0/3370784163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:28.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:28 smithi082 bash[27462]: cluster 2023-10-09T04:14:26.920400+0000 mgr.y (mgr.14436) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:28.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:28 smithi082 bash[27462]: audit 2023-10-09T04:14:27.462177+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.82:0/3370784163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:28.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:28 smithi082 bash[33901]: cluster 2023-10-09T04:14:26.920400+0000 mgr.y (mgr.14436) 203 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:28.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:28 smithi082 bash[33901]: audit 2023-10-09T04:14:27.462177+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.82:0/3370784163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:30 smithi153 bash[27868]: cluster 2023-10-09T04:14:28.921576+0000 mgr.y (mgr.14436) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:30 smithi153 bash[27868]: audit 2023-10-09T04:14:29.868010+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.82:0/745662422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:30 smithi082 bash[33901]: cluster 2023-10-09T04:14:28.921576+0000 mgr.y (mgr.14436) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:30 smithi082 bash[33901]: audit 2023-10-09T04:14:29.868010+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.82:0/745662422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:30.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:30 smithi082 bash[27462]: cluster 2023-10-09T04:14:28.921576+0000 mgr.y (mgr.14436) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:30.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:30 smithi082 bash[27462]: audit 2023-10-09T04:14:29.868010+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.82:0/745662422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:32.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:32 smithi153 bash[27868]: cluster 2023-10-09T04:14:30.922316+0000 mgr.y (mgr.14436) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:32.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:32 smithi082 bash[27462]: cluster 2023-10-09T04:14:30.922316+0000 mgr.y (mgr.14436) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:32 smithi082 bash[33901]: cluster 2023-10-09T04:14:30.922316+0000 mgr.y (mgr.14436) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:33.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:32] "GET /metrics HTTP/1.1" 200 192087 "" "Prometheus/2.33.4" 2023-10-09T04:14:33.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:33 smithi153 bash[27868]: audit 2023-10-09T04:14:32.276276+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.82:0/1545376861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:33 smithi082 bash[27462]: audit 2023-10-09T04:14:32.276276+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.82:0/1545376861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:33.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:33 smithi082 bash[33901]: audit 2023-10-09T04:14:32.276276+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.82:0/1545376861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:34.282 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:33 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:14:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:34 smithi153 bash[27868]: cluster 2023-10-09T04:14:32.923125+0000 mgr.y (mgr.14436) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:34 smithi082 bash[27462]: cluster 2023-10-09T04:14:32.923125+0000 mgr.y (mgr.14436) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:34.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:34 smithi082 bash[33901]: cluster 2023-10-09T04:14:32.923125+0000 mgr.y (mgr.14436) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:35 smithi153 bash[27868]: audit 2023-10-09T04:14:34.686809+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.82:0/903114783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:35.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:35 smithi082 bash[27462]: audit 2023-10-09T04:14:34.686809+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.82:0/903114783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:35.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:35 smithi082 bash[33901]: audit 2023-10-09T04:14:34.686809+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.82:0/903114783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:36 smithi153 bash[27868]: cluster 2023-10-09T04:14:34.923748+0000 mgr.y (mgr.14436) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:36 smithi082 bash[27462]: cluster 2023-10-09T04:14:34.923748+0000 mgr.y (mgr.14436) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:36.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:36 smithi082 bash[33901]: cluster 2023-10-09T04:14:34.923748+0000 mgr.y (mgr.14436) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:37 smithi153 bash[27868]: cluster 2023-10-09T04:14:36.924367+0000 mgr.y (mgr.14436) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:37 smithi153 bash[27868]: audit 2023-10-09T04:14:37.088885+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.82:0/3000815362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:37 smithi082 bash[33901]: cluster 2023-10-09T04:14:36.924367+0000 mgr.y (mgr.14436) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:37 smithi082 bash[33901]: audit 2023-10-09T04:14:37.088885+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.82:0/3000815362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:37 smithi082 bash[27462]: cluster 2023-10-09T04:14:36.924367+0000 mgr.y (mgr.14436) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:37 smithi082 bash[27462]: audit 2023-10-09T04:14:37.088885+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.82:0/3000815362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:40 smithi153 bash[27868]: cluster 2023-10-09T04:14:38.925474+0000 mgr.y (mgr.14436) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:40 smithi153 bash[27868]: audit 2023-10-09T04:14:39.500338+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.82:0/1244368653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:40 smithi082 bash[33901]: cluster 2023-10-09T04:14:38.925474+0000 mgr.y (mgr.14436) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:40.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:40 smithi082 bash[33901]: audit 2023-10-09T04:14:39.500338+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.82:0/1244368653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:40 smithi082 bash[27462]: cluster 2023-10-09T04:14:38.925474+0000 mgr.y (mgr.14436) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:40.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:40 smithi082 bash[27462]: audit 2023-10-09T04:14:39.500338+0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.82:0/1244368653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:42.292 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:42 smithi153 bash[27868]: cluster 2023-10-09T04:14:40.926058+0000 mgr.y (mgr.14436) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:42.292 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:42 smithi153 bash[27868]: audit 2023-10-09T04:14:41.916990+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.82:0/281880749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:42.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:42 smithi082 bash[33901]: cluster 2023-10-09T04:14:40.926058+0000 mgr.y (mgr.14436) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:42.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:42 smithi082 bash[33901]: audit 2023-10-09T04:14:41.916990+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.82:0/281880749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:42.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:42 smithi082 bash[27462]: cluster 2023-10-09T04:14:40.926058+0000 mgr.y (mgr.14436) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:42.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:42 smithi082 bash[27462]: audit 2023-10-09T04:14:41.916990+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.82:0/281880749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:43.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:42 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:42] "GET /metrics HTTP/1.1" 200 192084 "" "Prometheus/2.33.4" 2023-10-09T04:14:44.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[28717]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:14:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:14:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: cluster 2023-10-09T04:14:42.927027+0000 mgr.y (mgr.14436) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: cephadm 2023-10-09T04:14:42.963036+0000 mgr.y (mgr.14436) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:14:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: audit 2023-10-09T04:14:42.971670+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:44.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: audit 2023-10-09T04:14:42.973429+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:14:44.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: audit 2023-10-09T04:14:42.974991+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:14:44.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: audit 2023-10-09T04:14:42.976399+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:44.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:43 smithi153 bash[27868]: cephadm 2023-10-09T04:14:42.977572+0000 mgr.y (mgr.14436) 213 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:14:44.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: cluster 2023-10-09T04:14:42.927027+0000 mgr.y (mgr.14436) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:44.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: cephadm 2023-10-09T04:14:42.963036+0000 mgr.y (mgr.14436) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: audit 2023-10-09T04:14:42.971670+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: audit 2023-10-09T04:14:42.973429+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: audit 2023-10-09T04:14:42.974991+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: audit 2023-10-09T04:14:42.976399+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[27462]: cephadm 2023-10-09T04:14:42.977572+0000 mgr.y (mgr.14436) 213 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: cluster 2023-10-09T04:14:42.927027+0000 mgr.y (mgr.14436) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: cephadm 2023-10-09T04:14:42.963036+0000 mgr.y (mgr.14436) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: audit 2023-10-09T04:14:42.971670+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: audit 2023-10-09T04:14:42.973429+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:14:44.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: audit 2023-10-09T04:14:42.974991+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:14:44.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: audit 2023-10-09T04:14:42.976399+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:44.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:43 smithi082 bash[33901]: cephadm 2023-10-09T04:14:42.977572+0000 mgr.y (mgr.14436) 213 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:14:45.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:45 smithi153 bash[27868]: audit 2023-10-09T04:14:44.328577+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.82:0/1805002357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:45.135 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 systemd[1]: Stopping Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:14:45.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 bash[50582]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr.x 2023-10-09T04:14:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:45 smithi082 bash[27462]: audit 2023-10-09T04:14:44.328577+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.82:0/1805002357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:45 smithi082 bash[33901]: audit 2023-10-09T04:14:44.328577+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.82:0/1805002357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:46.125 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 bash[50617]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:14:46.125 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:14:46.125 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 bash[50682]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:14:46.125 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:14:46.125 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:45 smithi153 systemd[1]: Stopped Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:14:46.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:46 smithi153 systemd[1]: Started Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:14:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:46 smithi153 bash[27868]: cluster 2023-10-09T04:14:44.927646+0000 mgr.y (mgr.14436) 214 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:46 smithi082 bash[27462]: cluster 2023-10-09T04:14:44.927646+0000 mgr.y (mgr.14436) 214 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:46 smithi082 bash[33901]: cluster 2023-10-09T04:14:44.927646+0000 mgr.y (mgr.14436) 214 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:47.423 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:47 smithi153 bash[27868]: audit 2023-10-09T04:14:46.258663+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:47.423 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:47 smithi153 bash[27868]: audit 2023-10-09T04:14:46.259958+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:47.423 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:47 smithi153 bash[27868]: audit 2023-10-09T04:14:46.720306+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.82:0/2691856128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:47.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:47 smithi153 bash[50739]: debug 2023-10-09T04:14:47.416+0000 7f5417e8f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:14:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[27462]: audit 2023-10-09T04:14:46.258663+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[27462]: audit 2023-10-09T04:14:46.259958+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:47.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[27462]: audit 2023-10-09T04:14:46.720306+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.82:0/2691856128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[33901]: audit 2023-10-09T04:14:46.258663+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[33901]: audit 2023-10-09T04:14:46.259958+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:47 smithi082 bash[33901]: audit 2023-10-09T04:14:46.720306+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.82:0/2691856128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:48.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:48 smithi153 bash[27868]: cluster 2023-10-09T04:14:46.928307+0000 mgr.y (mgr.14436) 215 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:48.575 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:48 smithi153 bash[50739]: debug 2023-10-09T04:14:48.360+0000 7f5417e8f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:14:48.575 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:48 smithi153 bash[50739]: debug 2023-10-09T04:14:48.464+0000 7f5417e8f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:14:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:48 smithi082 bash[27462]: cluster 2023-10-09T04:14:46.928307+0000 mgr.y (mgr.14436) 215 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:48.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:48 smithi082 bash[33901]: cluster 2023-10-09T04:14:46.928307+0000 mgr.y (mgr.14436) 215 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:48.841 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:48 smithi153 bash[50739]: debug 2023-10-09T04:14:48.568+0000 7f5417e8f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:14:49.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:48 smithi153 bash[50739]: debug 2023-10-09T04:14:48.832+0000 7f5417e8f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:14:49.579 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:49 smithi153 bash[50739]: debug 2023-10-09T04:14:49.192+0000 7f5417e8f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:14:49.579 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:49 smithi153 bash[50739]: debug 2023-10-09T04:14:49.352+0000 7f5417e8f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:14:49.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:49 smithi153 bash[50739]: debug 2023-10-09T04:14:49.572+0000 7f5417e8f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:14:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:50 smithi153 bash[27868]: cluster 2023-10-09T04:14:48.929404+0000 mgr.y (mgr.14436) 216 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:50 smithi153 bash[27868]: audit 2023-10-09T04:14:49.120505+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.82:0/2764850163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:50 smithi082 bash[27462]: cluster 2023-10-09T04:14:48.929404+0000 mgr.y (mgr.14436) 216 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:50.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:50 smithi082 bash[27462]: audit 2023-10-09T04:14:49.120505+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.82:0/2764850163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:50.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:50 smithi082 bash[33901]: cluster 2023-10-09T04:14:48.929404+0000 mgr.y (mgr.14436) 216 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:50.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:50 smithi082 bash[33901]: audit 2023-10-09T04:14:49.120505+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.82:0/2764850163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:50.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:50 smithi153 bash[50739]: debug 2023-10-09T04:14:50.596+0000 7f5417e8f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:14:50.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:50 smithi153 bash[50739]: debug 2023-10-09T04:14:50.712+0000 7f5417e8f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:14:50.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:50 smithi153 bash[50739]: debug 2023-10-09T04:14:50.832+0000 7f5417e8f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:14:51.621 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:51 smithi153 bash[50739]: debug 2023-10-09T04:14:51.332+0000 7f5417e8f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:14:51.621 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:51 smithi153 bash[50739]: debug 2023-10-09T04:14:51.612+0000 7f5417e8f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:14:52.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:52 smithi153 bash[50739]: debug 2023-10-09T04:14:52.148+0000 7f5417e8f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:14:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:52 smithi153 bash[27868]: cluster 2023-10-09T04:14:50.930000+0000 mgr.y (mgr.14436) 217 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:52 smithi153 bash[27868]: audit 2023-10-09T04:14:51.545755+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.82:0/768152168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:52.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:52 smithi153 bash[27868]: audit 2023-10-09T04:14:51.962634+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:52.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27462]: cluster 2023-10-09T04:14:50.930000+0000 mgr.y (mgr.14436) 217 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:52.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27462]: audit 2023-10-09T04:14:51.545755+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.82:0/768152168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:52.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27462]: audit 2023-10-09T04:14:51.962634+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:52.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[33901]: cluster 2023-10-09T04:14:50.930000+0000 mgr.y (mgr.14436) 217 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:52.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[33901]: audit 2023-10-09T04:14:51.545755+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.82:0/768152168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:52.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[33901]: audit 2023-10-09T04:14:51.962634+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:52.967 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: debug 2023-10-09T04:14:52.642+0000 7f1120dfb700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T04:14:52.967 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: ignoring --setuser ceph since I am not root 2023-10-09T04:14:52.967 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: ignoring --setgroup ceph since I am not root 2023-10-09T04:14:52.967 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: debug 2023-10-09T04:14:52.698+0000 7fc9c6502700 1 -- 172.21.15.82:0/71458796 <== mon.0 v2:172.21.15.82:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557c74ba04e0 con 0x557c74c94c00 2023-10-09T04:14:52.967 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: debug 2023-10-09T04:14:52.854+0000 7fc9d66ca680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:14:53.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:52 smithi153 bash[50739]: debug 2023-10-09T04:14:52.692+0000 7f5417e8f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:14:53.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:52 smithi082 bash[27740]: debug 2023-10-09T04:14:52.958+0000 7fc9d66ca680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:14:53.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[50739]: debug 2023-10-09T04:14:53.248+0000 7f5417e8f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:14:53.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[50739]: debug 2023-10-09T04:14:53.360+0000 7f5417e8f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:14:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.357833+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.614997+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.617034+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.625441+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.633530+0000 mon.a (mon.0) 786 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.640425+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:14:53.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: cluster 2023-10-09T04:14:52.650353+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:14:53.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: audit 2023-10-09T04:14:52.651019+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:14:53.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[27868]: cluster 2023-10-09T04:14:52.651135+0000 mon.a (mon.0) 790 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00988561s) 2023-10-09T04:14:53.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27740]: debug 2023-10-09T04:14:53.442+0000 7fc9d66ca680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:14:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.357833+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.614997+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.617034+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.625441+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.633530+0000 mon.a (mon.0) 786 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:53.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.640425+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:14:53.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: cluster 2023-10-09T04:14:52.650353+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:14:53.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: audit 2023-10-09T04:14:52.651019+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:14:53.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[33901]: cluster 2023-10-09T04:14:52.651135+0000 mon.a (mon.0) 790 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00988561s) 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.357833+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.614997+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.617034+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.625441+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.633530+0000 mon.a (mon.0) 786 : audit [DBG] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:53.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.640425+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:14:53.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: cluster 2023-10-09T04:14:52.650353+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:14:53.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: audit 2023-10-09T04:14:52.651019+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.14436 172.21.15.82:0/4032272513' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:14:53.736 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:53 smithi082 bash[27462]: cluster 2023-10-09T04:14:52.651135+0000 mon.a (mon.0) 790 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00988561s) 2023-10-09T04:14:53.978 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[50739]: debug 2023-10-09T04:14:53.708+0000 7f5417e8f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:14:53.978 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[50739]: debug 2023-10-09T04:14:53.820+0000 7f5417e8f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:14:54.364 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:53 smithi153 bash[50739]: debug 2023-10-09T04:14:53.972+0000 7f5417e8f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:14:54.364 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:54 smithi153 bash[50739]: debug 2023-10-09T04:14:54.088+0000 7f5417e8f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:14:54.364 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:54 smithi153 bash[50739]: debug 2023-10-09T04:14:54.220+0000 7f5417e8f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:14:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:54 smithi153 bash[27868]: audit 2023-10-09T04:14:53.969604+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.82:0/891070527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:54.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:54 smithi153 bash[50739]: debug 2023-10-09T04:14:54.416+0000 7f5417e8f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:14:54.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:54 smithi153 bash[50739]: debug 2023-10-09T04:14:54.548+0000 7f5417e8f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:14:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:54 smithi082 bash[27462]: audit 2023-10-09T04:14:53.969604+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.82:0/891070527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:54 smithi082 bash[33901]: audit 2023-10-09T04:14:53.969604+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.82:0/891070527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:54.733 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:54 smithi082 bash[27740]: debug 2023-10-09T04:14:54.482+0000 7fc9d66ca680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:14:54.733 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:54 smithi082 bash[27740]: debug 2023-10-09T04:14:54.586+0000 7fc9d66ca680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:14:55.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:54 smithi082 bash[27740]: debug 2023-10-09T04:14:54.862+0000 7fc9d66ca680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:14:56.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: cluster 2023-10-09T04:14:55.807406+0000 mon.a (mon.0) 792 : cluster [INF] Active manager daemon x restarted 2023-10-09T04:14:56.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: cluster 2023-10-09T04:14:55.808031+0000 mon.a (mon.0) 793 : cluster [INF] Activating manager daemon x 2023-10-09T04:14:56.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.817673+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: cluster 2023-10-09T04:14:55.818042+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:14:56.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: cluster 2023-10-09T04:14:55.818264+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0104823s) 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.819643+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.819918+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.820175+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.820433+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.820870+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.821191+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.821522+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.821812+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:14:56.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.822158+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.822479+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.822818+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.823138+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.823469+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.823794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.824145+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.824464+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.824769+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.825253+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.825489+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.826471+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:14:56.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[27868]: audit 2023-10-09T04:14:55.826967+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: debug 2023-10-09T04:14:55.784+0000 7f5417e8f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE Bus STARTING 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: CherryPy Checker: 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: The Application mounted at '' has an empty config. 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE Serving on http://:::9283 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE Bus STARTED 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE Bus STOPPING 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:14:56.100 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:55 smithi153 bash[50739]: [09/Oct/2023:04:14:55] ENGINE Bus STOPPED 2023-10-09T04:14:56.101 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: [09/Oct/2023:04:14:56] ENGINE Bus STARTING 2023-10-09T04:14:56.180 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27740]: debug 2023-10-09T04:14:56.062+0000 7fc9d66ca680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: cluster 2023-10-09T04:14:55.807406+0000 mon.a (mon.0) 792 : cluster [INF] Active manager daemon x restarted 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: cluster 2023-10-09T04:14:55.808031+0000 mon.a (mon.0) 793 : cluster [INF] Activating manager daemon x 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.817673+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: cluster 2023-10-09T04:14:55.818042+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: cluster 2023-10-09T04:14:55.818264+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0104823s) 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.819643+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.819918+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.820175+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.820433+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.820870+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.821191+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.821522+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.821812+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.822158+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.822479+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.822818+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.823138+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:14:56.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.823469+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.823794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: cluster 2023-10-09T04:14:55.807406+0000 mon.a (mon.0) 792 : cluster [INF] Active manager daemon x restarted 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: cluster 2023-10-09T04:14:55.808031+0000 mon.a (mon.0) 793 : cluster [INF] Activating manager daemon x 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.817673+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: cluster 2023-10-09T04:14:55.818042+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: cluster 2023-10-09T04:14:55.818264+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0104823s) 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.819643+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:14:56.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.819918+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.820175+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.820433+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.820870+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.821191+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.821522+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.821812+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.822158+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.822479+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.822818+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.823138+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:14:56.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.823469+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:14:56.185 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.823794+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:14:56.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.824145+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:14:56.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.824464+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:14:56.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.824769+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.825253+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.825489+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.826471+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[33901]: audit 2023-10-09T04:14:55.826967+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.824145+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:14:56.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.824464+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:14:56.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.824769+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:14:56.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.825253+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:14:56.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.825489+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:14:56.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.826471+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:14:56.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:55 smithi082 bash[27462]: audit 2023-10-09T04:14:55.826967+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5375329700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5375329700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5375329700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5375329700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5375329700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5379b32700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5379b32700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5379b32700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5379b32700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: debug 2023-10-09T04:14:56.180+0000 7f5379b32700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: [09/Oct/2023:04:14:56] ENGINE Serving on http://:::9283 2023-10-09T04:14:56.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[50739]: [09/Oct/2023:04:14:56] ENGINE Bus STARTED 2023-10-09T04:14:56.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27740]: debug 2023-10-09T04:14:56.238+0000 7fc9d66ca680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:14:56.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27740]: debug 2023-10-09T04:14:56.350+0000 7fc9d66ca680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:14:56.870 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27740]: debug 2023-10-09T04:14:56.474+0000 7fc9d66ca680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:14:56.870 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27740]: debug 2023-10-09T04:14:56.726+0000 7fc9d66ca680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: cluster 2023-10-09T04:14:56.005847+0000 mon.a (mon.0) 796 : cluster [INF] Manager daemon x is now available 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.041695+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.054276+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.061135+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.070702+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.070883+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.092778+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.113505+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.113569+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.142054+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.150169+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.150198+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.379072+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.82:0/3463083505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.536538+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[33901]: audit 2023-10-09T04:14:56.545747+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: cluster 2023-10-09T04:14:56.005847+0000 mon.a (mon.0) 796 : cluster [INF] Manager daemon x is now available 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.041695+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.054276+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.061135+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.070702+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.070883+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.092778+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.113505+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.113569+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.142054+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.150169+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.150198+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.379072+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.82:0/3463083505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:57.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.536538+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.141 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:56 smithi082 bash[27462]: audit 2023-10-09T04:14:56.545747+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: cluster 2023-10-09T04:14:56.005847+0000 mon.a (mon.0) 796 : cluster [INF] Manager daemon x is now available 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.041695+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.054276+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.061135+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.070702+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.070883+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.092778+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.113505+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.113569+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.142054+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.150169+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.150198+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.379072+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.82:0/3463083505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.536538+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:56 smithi153 bash[27868]: audit 2023-10-09T04:14:56.545747+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:14:57.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27740]: debug 2023-10-09T04:14:57.130+0000 7fc9d66ca680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:14:57.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27740]: debug 2023-10-09T04:14:57.254+0000 7fc9d66ca680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: cephadm 2023-10-09T04:14:56.600090+0000 mgr.x (mgr.25177) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: cephadm 2023-10-09T04:14:56.604840+0000 mgr.x (mgr.25177) 2 : cephadm [INF] Deploying cephadm binary to smithi082 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: cluster 2023-10-09T04:14:56.884162+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e21: x(active, since 1.07638s) 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: cluster 2023-10-09T04:14:56.933301+0000 mgr.x (mgr.25177) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: audit 2023-10-09T04:14:57.046234+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: cluster 2023-10-09T04:14:57.046385+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:14:58.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: audit 2023-10-09T04:14:57.058232+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[27462]: audit 2023-10-09T04:14:57.058518+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: cephadm 2023-10-09T04:14:56.600090+0000 mgr.x (mgr.25177) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: cephadm 2023-10-09T04:14:56.604840+0000 mgr.x (mgr.25177) 2 : cephadm [INF] Deploying cephadm binary to smithi082 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: cluster 2023-10-09T04:14:56.884162+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e21: x(active, since 1.07638s) 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: cluster 2023-10-09T04:14:56.933301+0000 mgr.x (mgr.25177) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: audit 2023-10-09T04:14:57.046234+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: cluster 2023-10-09T04:14:57.046385+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: audit 2023-10-09T04:14:57.058232+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:57 smithi082 bash[33901]: audit 2023-10-09T04:14:57.058518+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: cephadm 2023-10-09T04:14:56.600090+0000 mgr.x (mgr.25177) 1 : cephadm [INF] Deploying cephadm binary to smithi153 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: cephadm 2023-10-09T04:14:56.604840+0000 mgr.x (mgr.25177) 2 : cephadm [INF] Deploying cephadm binary to smithi082 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: cluster 2023-10-09T04:14:56.884162+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e21: x(active, since 1.07638s) 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: cluster 2023-10-09T04:14:56.933301+0000 mgr.x (mgr.25177) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: audit 2023-10-09T04:14:57.046234+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: cluster 2023-10-09T04:14:57.046385+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:14:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: audit 2023-10-09T04:14:57.058232+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:57 smithi153 bash[27868]: audit 2023-10-09T04:14:57.058518+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:14:58.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:58 smithi082 bash[27740]: debug 2023-10-09T04:14:58.134+0000 7fc9d66ca680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:14:58.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:58 smithi082 bash[27740]: debug 2023-10-09T04:14:58.246+0000 7fc9d66ca680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:14:58.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:58 smithi082 bash[27740]: debug 2023-10-09T04:14:58.374+0000 7fc9d66ca680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:14:58.924 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:58 smithi082 bash[27740]: debug 2023-10-09T04:14:58.614+0000 7fc9d66ca680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:14:59.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:58 smithi082 bash[27740]: debug 2023-10-09T04:14:58.918+0000 7fc9d66ca680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: debug 2023-10-09T04:14:59.098+0000 7fc9d66ca680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cephadm 2023-10-09T04:14:57.413688+0000 mgr.x (mgr.25177) 4 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTING 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cephadm 2023-10-09T04:14:57.654388+0000 mgr.x (mgr.25177) 5 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on https://172.21.15.153:7150 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cephadm 2023-10-09T04:14:57.759965+0000 mgr.x (mgr.25177) 6 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on http://172.21.15.153:8765 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cephadm 2023-10-09T04:14:57.760465+0000 mgr.x (mgr.25177) 7 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTED 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cluster 2023-10-09T04:14:57.824264+0000 mgr.x (mgr.25177) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:59.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cluster 2023-10-09T04:14:58.049718+0000 mon.a (mon.0) 808 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: audit 2023-10-09T04:14:58.056999+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: cluster 2023-10-09T04:14:58.057141+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[33901]: audit 2023-10-09T04:14:58.804972+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.82:0/1388827643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cephadm 2023-10-09T04:14:57.413688+0000 mgr.x (mgr.25177) 4 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTING 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cephadm 2023-10-09T04:14:57.654388+0000 mgr.x (mgr.25177) 5 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on https://172.21.15.153:7150 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cephadm 2023-10-09T04:14:57.759965+0000 mgr.x (mgr.25177) 6 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on http://172.21.15.153:8765 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cephadm 2023-10-09T04:14:57.760465+0000 mgr.x (mgr.25177) 7 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTED 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cluster 2023-10-09T04:14:57.824264+0000 mgr.x (mgr.25177) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:59.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cluster 2023-10-09T04:14:58.049718+0000 mon.a (mon.0) 808 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T04:14:59.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: audit 2023-10-09T04:14:58.056999+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:59.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: cluster 2023-10-09T04:14:58.057141+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:14:59.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27462]: audit 2023-10-09T04:14:58.804972+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.82:0/1388827643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cephadm 2023-10-09T04:14:57.413688+0000 mgr.x (mgr.25177) 4 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTING 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cephadm 2023-10-09T04:14:57.654388+0000 mgr.x (mgr.25177) 5 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on https://172.21.15.153:7150 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cephadm 2023-10-09T04:14:57.759965+0000 mgr.x (mgr.25177) 6 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Serving on http://172.21.15.153:8765 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cephadm 2023-10-09T04:14:57.760465+0000 mgr.x (mgr.25177) 7 : cephadm [INF] [09/Oct/2023:04:14:57] ENGINE Bus STARTED 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cluster 2023-10-09T04:14:57.824264+0000 mgr.x (mgr.25177) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:14:59.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cluster 2023-10-09T04:14:58.049718+0000 mon.a (mon.0) 808 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T04:14:59.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: audit 2023-10-09T04:14:58.056999+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:14:59.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: cluster 2023-10-09T04:14:58.057141+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:14:59.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:14:59 smithi153 bash[27868]: audit 2023-10-09T04:14:58.804972+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.82:0/1388827643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:14:59.641 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: debug 2023-10-09T04:14:59.290+0000 7fc9d66ca680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:14:59.641 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: debug 2023-10-09T04:14:59.522+0000 7fc9d66ca680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:14:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: debug 2023-10-09T04:14:59.634+0000 7fc9d66ca680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:14:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: [09/Oct/2023:04:14:59] ENGINE Bus STARTING 2023-10-09T04:14:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: CherryPy Checker: 2023-10-09T04:14:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: The Application mounted at '' has an empty config. 2023-10-09T04:14:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: [09/Oct/2023:04:14:59] ENGINE Serving on http://:::9283 2023-10-09T04:14:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:14:59 smithi082 bash[27740]: [09/Oct/2023:04:14:59] ENGINE Bus STARTED 2023-10-09T04:15:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: audit 2023-10-09T04:14:59.282618+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: cluster 2023-10-09T04:14:59.640993+0000 mon.a (mon.0) 812 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:15:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: audit 2023-10-09T04:14:59.644626+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:15:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: audit 2023-10-09T04:14:59.645389+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:15:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: audit 2023-10-09T04:14:59.648261+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:15:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: audit 2023-10-09T04:14:59.648685+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:15:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:00 smithi153 bash[27868]: cluster 2023-10-09T04:14:59.824855+0000 mgr.x (mgr.25177) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:00.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: audit 2023-10-09T04:14:59.282618+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:00.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: cluster 2023-10-09T04:14:59.640993+0000 mon.a (mon.0) 812 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:15:00.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: audit 2023-10-09T04:14:59.644626+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:15:00.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: audit 2023-10-09T04:14:59.645389+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:15:00.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: audit 2023-10-09T04:14:59.648261+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: audit 2023-10-09T04:14:59.648685+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[27462]: cluster 2023-10-09T04:14:59.824855+0000 mgr.x (mgr.25177) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: audit 2023-10-09T04:14:59.282618+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: cluster 2023-10-09T04:14:59.640993+0000 mon.a (mon.0) 812 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: audit 2023-10-09T04:14:59.644626+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:15:00.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: audit 2023-10-09T04:14:59.645389+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:15:00.736 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: audit 2023-10-09T04:14:59.648261+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:15:00.736 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: audit 2023-10-09T04:14:59.648685+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.? 172.21.15.82:0/1438564038' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:15:00.736 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:00 smithi082 bash[33901]: cluster 2023-10-09T04:14:59.824855+0000 mgr.x (mgr.25177) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:01 smithi153 bash[27868]: cluster 2023-10-09T04:15:00.293181+0000 mon.a (mon.0) 817 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T04:15:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:01 smithi153 bash[27868]: audit 2023-10-09T04:15:00.296684+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:15:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:01 smithi153 bash[27868]: audit 2023-10-09T04:15:01.238146+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.82:0/1575409794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[27462]: cluster 2023-10-09T04:15:00.293181+0000 mon.a (mon.0) 817 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T04:15:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[27462]: audit 2023-10-09T04:15:00.296684+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:15:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[27462]: audit 2023-10-09T04:15:01.238146+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.82:0/1575409794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[33901]: cluster 2023-10-09T04:15:00.293181+0000 mon.a (mon.0) 817 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T04:15:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[33901]: audit 2023-10-09T04:15:00.296684+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:15:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:01 smithi082 bash[33901]: audit 2023-10-09T04:15:01.238146+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.82:0/1575409794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:02 smithi153 bash[27868]: cluster 2023-10-09T04:15:01.825463+0000 mgr.x (mgr.25177) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:02.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:02 smithi082 bash[27462]: cluster 2023-10-09T04:15:01.825463+0000 mgr.x (mgr.25177) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:02.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:02 smithi082 bash[33901]: cluster 2023-10-09T04:15:01.825463+0000 mgr.x (mgr.25177) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:03.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:15:02 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:15:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:04 smithi153 bash[27868]: audit 2023-10-09T04:15:03.002825+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:04 smithi153 bash[27868]: audit 2023-10-09T04:15:03.682122+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.82:0/1357435108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:04.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:15:03 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:03] "GET /metrics HTTP/1.1" 200 30893 "" "Prometheus/2.33.4" 2023-10-09T04:15:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:04 smithi082 bash[33901]: audit 2023-10-09T04:15:03.002825+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:04 smithi082 bash[33901]: audit 2023-10-09T04:15:03.682122+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.82:0/1357435108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:04.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:04 smithi082 bash[27462]: audit 2023-10-09T04:15:03.002825+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:04.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:04 smithi082 bash[27462]: audit 2023-10-09T04:15:03.682122+0000 mon.a (mon.0) 819 : audit [DBG] from='client.? 172.21.15.82:0/1357435108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:05 smithi153 bash[27868]: cluster 2023-10-09T04:15:03.826645+0000 mgr.x (mgr.25177) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 31 op/s 2023-10-09T04:15:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:05 smithi153 bash[27868]: audit 2023-10-09T04:15:04.871974+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:05 smithi153 bash[27868]: audit 2023-10-09T04:15:04.880621+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[27462]: cluster 2023-10-09T04:15:03.826645+0000 mgr.x (mgr.25177) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 31 op/s 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[27462]: audit 2023-10-09T04:15:04.871974+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[27462]: audit 2023-10-09T04:15:04.880621+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[33901]: cluster 2023-10-09T04:15:03.826645+0000 mgr.x (mgr.25177) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 31 op/s 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[33901]: audit 2023-10-09T04:15:04.871974+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:05 smithi082 bash[33901]: audit 2023-10-09T04:15:04.880621+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:06 smithi153 bash[27868]: audit 2023-10-09T04:15:05.197705+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:06 smithi153 bash[27868]: audit 2023-10-09T04:15:05.207014+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:06 smithi153 bash[27868]: audit 2023-10-09T04:15:05.731368+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:06 smithi153 bash[27868]: audit 2023-10-09T04:15:05.740211+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:06 smithi153 bash[27868]: audit 2023-10-09T04:15:05.812966+0000 mon.a (mon.0) 826 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[27462]: audit 2023-10-09T04:15:05.197705+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[27462]: audit 2023-10-09T04:15:05.207014+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[27462]: audit 2023-10-09T04:15:05.731368+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[27462]: audit 2023-10-09T04:15:05.740211+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[27462]: audit 2023-10-09T04:15:05.812966+0000 mon.a (mon.0) 826 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[33901]: audit 2023-10-09T04:15:05.197705+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[33901]: audit 2023-10-09T04:15:05.207014+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[33901]: audit 2023-10-09T04:15:05.731368+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[33901]: audit 2023-10-09T04:15:05.740211+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:07.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:06 smithi082 bash[33901]: audit 2023-10-09T04:15:05.812966+0000 mon.a (mon.0) 826 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:08.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:05.813969+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:05.814369+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:05.814767+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: cluster 2023-10-09T04:15:05.827213+0000 mgr.x (mgr.25177) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:06.131277+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.82:0/1395982027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:06.736542+0000 mon.a (mon.0) 827 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:06.746335+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.680854+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.682335+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.682768+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.686582+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.688930+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.696420+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.696587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.696713+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:08 smithi153 bash[27868]: audit 2023-10-09T04:15:07.703071+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:05.813969+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:05.814369+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:05.814767+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: cluster 2023-10-09T04:15:05.827213+0000 mgr.x (mgr.25177) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:06.131277+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.82:0/1395982027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:06.736542+0000 mon.a (mon.0) 827 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:06.746335+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.680854+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.682335+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.682768+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.686582+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.688930+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.696420+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.696587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.696713+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[27462]: audit 2023-10-09T04:15:07.703071+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:05.813969+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:05.814369+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:05.814767+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: cluster 2023-10-09T04:15:05.827213+0000 mgr.x (mgr.25177) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:06.131277+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.82:0/1395982027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:06.736542+0000 mon.a (mon.0) 827 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:06.746335+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.680854+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.682335+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.682768+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.686582+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.688930+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.696420+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:15:08.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.696587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:15:08.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.696713+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:15:08.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:08 smithi082 bash[33901]: audit 2023-10-09T04:15:07.703071+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:09 smithi153 bash[27868]: audit 2023-10-09T04:15:07.697932+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:09 smithi153 bash[27868]: cluster 2023-10-09T04:15:07.828279+0000 mgr.x (mgr.25177) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T04:15:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:09 smithi153 bash[27868]: audit 2023-10-09T04:15:08.567256+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.82:0/4150021639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:09 smithi153 bash[27868]: audit 2023-10-09T04:15:08.629512+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[27462]: audit 2023-10-09T04:15:07.697932+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[27462]: cluster 2023-10-09T04:15:07.828279+0000 mgr.x (mgr.25177) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[27462]: audit 2023-10-09T04:15:08.567256+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.82:0/4150021639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[27462]: audit 2023-10-09T04:15:08.629512+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[33901]: audit 2023-10-09T04:15:07.697932+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[33901]: cluster 2023-10-09T04:15:07.828279+0000 mgr.x (mgr.25177) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[33901]: audit 2023-10-09T04:15:08.567256+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.82:0/4150021639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:09 smithi082 bash[33901]: audit 2023-10-09T04:15:08.629512+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:15:10.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:10 smithi153 bash[27868]: cluster 2023-10-09T04:15:09.829018+0000 mgr.x (mgr.25177) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T04:15:10.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:10 smithi082 bash[27462]: cluster 2023-10-09T04:15:09.829018+0000 mgr.x (mgr.25177) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T04:15:10.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:10 smithi082 bash[33901]: cluster 2023-10-09T04:15:09.829018+0000 mgr.x (mgr.25177) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T04:15:11.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:11 smithi082 bash[27462]: audit 2023-10-09T04:15:11.009011+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.82:0/2682313889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:11.920 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:11 smithi082 bash[27462]: audit 2023-10-09T04:15:11.093379+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:11.920 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:11 smithi082 bash[33901]: audit 2023-10-09T04:15:11.009011+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.82:0/2682313889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:11.920 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:11 smithi082 bash[33901]: audit 2023-10-09T04:15:11.093379+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:11 smithi153 bash[27868]: audit 2023-10-09T04:15:11.009011+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.82:0/2682313889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:11 smithi153 bash[27868]: audit 2023-10-09T04:15:11.093379+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:12.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:12 smithi082 bash[27462]: cluster 2023-10-09T04:15:11.829894+0000 mgr.x (mgr.25177) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T04:15:12.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:12 smithi082 bash[33901]: cluster 2023-10-09T04:15:11.829894+0000 mgr.x (mgr.25177) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T04:15:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:12 smithi153 bash[27868]: cluster 2023-10-09T04:15:11.829894+0000 mgr.x (mgr.25177) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T04:15:13.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:15:12 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:15:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:13 smithi153 bash[27868]: audit 2023-10-09T04:15:13.449700+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.82:0/3317357684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:13 smithi082 bash[33901]: audit 2023-10-09T04:15:13.449700+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.82:0/3317357684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:13.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:13 smithi082 bash[27462]: audit 2023-10-09T04:15:13.449700+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.82:0/3317357684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:14.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:15:13 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:13] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: cluster 2023-10-09T04:15:13.831301+0000 mgr.x (mgr.25177) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.857303+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.866019+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.870476+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.870504+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.872632+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.872649+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.388 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.874618+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.388 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.874817+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.388 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.876740+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.388 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.876824+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.388 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:15 smithi153 bash[27868]: audit 2023-10-09T04:15:14.887742+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: cluster 2023-10-09T04:15:13.831301+0000 mgr.x (mgr.25177) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.857303+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.866019+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.870476+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.870504+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.872632+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.872649+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.874618+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.874817+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.876740+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.876824+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[27462]: audit 2023-10-09T04:15:14.887742+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: cluster 2023-10-09T04:15:13.831301+0000 mgr.x (mgr.25177) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.857303+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.866019+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.870476+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.870504+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.872632+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.872649+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.874618+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.874817+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.876740+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.876824+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:15.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:15 smithi082 bash[33901]: audit 2023-10-09T04:15:14.887742+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:16 smithi153 bash[27868]: cephadm 2023-10-09T04:15:14.878599+0000 mgr.x (mgr.25177) 17 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 2524M 2023-10-09T04:15:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:16 smithi153 bash[27868]: audit 2023-10-09T04:15:15.902298+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.82:0/2239656734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:16 smithi082 bash[27462]: cephadm 2023-10-09T04:15:14.878599+0000 mgr.x (mgr.25177) 17 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 2524M 2023-10-09T04:15:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:16 smithi082 bash[27462]: audit 2023-10-09T04:15:15.902298+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.82:0/2239656734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:16 smithi082 bash[33901]: cephadm 2023-10-09T04:15:14.878599+0000 mgr.x (mgr.25177) 17 : cephadm [INF] Adjusting osd_memory_target on smithi082 to 2524M 2023-10-09T04:15:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:16 smithi082 bash[33901]: audit 2023-10-09T04:15:15.902298+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.82:0/2239656734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:17.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: cluster 2023-10-09T04:15:15.832135+0000 mgr.x (mgr.25177) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:17.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.911463+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.921331+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.925094+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.925359+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.927688+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.927764+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.929790+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.929850+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.932036+0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.932145+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.940184+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.943303+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:17.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:17 smithi153 bash[27868]: audit 2023-10-09T04:15:16.944779+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: cluster 2023-10-09T04:15:15.832135+0000 mgr.x (mgr.25177) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.911463+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.921331+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.925094+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.925359+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.927688+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.927764+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.929790+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.929850+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.932036+0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.932145+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.940184+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.943303+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[27462]: audit 2023-10-09T04:15:16.944779+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:17.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: cluster 2023-10-09T04:15:15.832135+0000 mgr.x (mgr.25177) 18 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.911463+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.921331+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.925094+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.925359+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.927688+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.927764+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.929790+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.929850+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.932036+0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.932145+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.940184+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:17.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.943303+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:17.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:17 smithi082 bash[33901]: audit 2023-10-09T04:15:16.944779+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: cephadm 2023-10-09T04:15:16.934059+0000 mgr.x (mgr.25177) 19 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2524M 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: cephadm 2023-10-09T04:15:16.946448+0000 mgr.x (mgr.25177) 20 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: cephadm 2023-10-09T04:15:16.946780+0000 mgr.x (mgr.25177) 21 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.161997+0000 mgr.x (mgr.25177) 22 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.162392+0000 mgr.x (mgr.25177) 23 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.666211+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.670558+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.677037+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.682237+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.689358+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.769236+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.775887+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.780804+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[27462]: audit 2023-10-09T04:15:17.786026+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.413 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:18 smithi082 systemd[1]: Stopping Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:18.413 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[71732]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter.a 2023-10-09T04:15:18.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: cephadm 2023-10-09T04:15:16.934059+0000 mgr.x (mgr.25177) 19 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2524M 2023-10-09T04:15:18.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: cephadm 2023-10-09T04:15:16.946448+0000 mgr.x (mgr.25177) 20 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:15:18.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: cephadm 2023-10-09T04:15:16.946780+0000 mgr.x (mgr.25177) 21 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.161997+0000 mgr.x (mgr.25177) 22 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.162392+0000 mgr.x (mgr.25177) 23 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.666211+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.670558+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.677037+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.682237+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.689358+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.769236+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.775887+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.780804+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[33901]: audit 2023-10-09T04:15:17.786026+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: cephadm 2023-10-09T04:15:16.934059+0000 mgr.x (mgr.25177) 19 : cephadm [INF] Adjusting osd_memory_target on smithi153 to 2524M 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: cephadm 2023-10-09T04:15:16.946448+0000 mgr.x (mgr.25177) 20 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: cephadm 2023-10-09T04:15:16.946780+0000 mgr.x (mgr.25177) 21 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.161997+0000 mgr.x (mgr.25177) 22 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.162392+0000 mgr.x (mgr.25177) 23 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:15:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.666211+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.670558+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.677037+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.682237+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.689358+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.769236+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.775887+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.780804+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:18.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:18 smithi153 bash[27868]: audit 2023-10-09T04:15:17.786026+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.084 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:18 smithi082 bash[71770]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T04:15:19.085 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:19 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:15:19.085 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[71836]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T04:15:19.085 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:19 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T04:15:19.085 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:19 smithi082 systemd[1]: Stopped Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:19.085 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:19 smithi082 systemd[1]: Started Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:19.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.326662+0000 mgr.x (mgr.25177) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.327399+0000 mgr.x (mgr.25177) 25 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.491653+0000 mgr.x (mgr.25177) 26 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.492028+0000 mgr.x (mgr.25177) 27 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.788591+0000 mgr.x (mgr.25177) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cephadm 2023-10-09T04:15:17.789333+0000 mgr.x (mgr.25177) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi082 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: cluster 2023-10-09T04:15:17.833346+0000 mgr.x (mgr.25177) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: audit 2023-10-09T04:15:18.355692+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.82:0/331973219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: audit 2023-10-09T04:15:19.064791+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:19 smithi153 bash[27868]: audit 2023-10-09T04:15:19.072512+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.326662+0000 mgr.x (mgr.25177) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.327399+0000 mgr.x (mgr.25177) 25 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.491653+0000 mgr.x (mgr.25177) 26 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.492028+0000 mgr.x (mgr.25177) 27 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.788591+0000 mgr.x (mgr.25177) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cephadm 2023-10-09T04:15:17.789333+0000 mgr.x (mgr.25177) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi082 2023-10-09T04:15:19.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: cluster 2023-10-09T04:15:17.833346+0000 mgr.x (mgr.25177) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: audit 2023-10-09T04:15:18.355692+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.82:0/331973219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: audit 2023-10-09T04:15:19.064791+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[27462]: audit 2023-10-09T04:15:19.072512+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.326662+0000 mgr.x (mgr.25177) 24 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.327399+0000 mgr.x (mgr.25177) 25 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.491653+0000 mgr.x (mgr.25177) 26 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.492028+0000 mgr.x (mgr.25177) 27 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:15:19.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.788591+0000 mgr.x (mgr.25177) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:15:19.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cephadm 2023-10-09T04:15:17.789333+0000 mgr.x (mgr.25177) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi082 2023-10-09T04:15:19.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: cluster 2023-10-09T04:15:17.833346+0000 mgr.x (mgr.25177) 30 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:15:19.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: audit 2023-10-09T04:15:18.355692+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.82:0/331973219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:19.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: audit 2023-10-09T04:15:19.064791+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:19.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:19 smithi082 bash[33901]: audit 2023-10-09T04:15:19.072512+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:20 smithi153 bash[27868]: cephadm 2023-10-09T04:15:19.108538+0000 mgr.x (mgr.25177) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:15:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:20 smithi153 bash[27868]: cephadm 2023-10-09T04:15:19.135188+0000 mgr.x (mgr.25177) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:15:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[27462]: cephadm 2023-10-09T04:15:19.108538+0000 mgr.x (mgr.25177) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:15:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[27462]: cephadm 2023-10-09T04:15:19.135188+0000 mgr.x (mgr.25177) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:15:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[33901]: cephadm 2023-10-09T04:15:19.108538+0000 mgr.x (mgr.25177) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:15:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[33901]: cephadm 2023-10-09T04:15:19.135188+0000 mgr.x (mgr.25177) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi082 2023-10-09T04:15:20.483 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.364Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T04:15:20.483 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.364Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T04:15:20.483 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T04:15:20.484 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.365Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T04:15:20.485 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T04:15:20.486 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T04:15:20.487 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 04:15:20 smithi082 bash[71865]: ts=2023-10-09T04:15:20.366Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T04:15:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:21 smithi153 bash[27868]: cluster 2023-10-09T04:15:19.834160+0000 mgr.x (mgr.25177) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:21 smithi153 bash[27868]: audit 2023-10-09T04:15:20.829991+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.82:0/249742919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:21.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:21 smithi082 bash[33901]: cluster 2023-10-09T04:15:19.834160+0000 mgr.x (mgr.25177) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:21.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:21 smithi082 bash[33901]: audit 2023-10-09T04:15:20.829991+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.82:0/249742919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:21 smithi082 bash[27462]: cluster 2023-10-09T04:15:19.834160+0000 mgr.x (mgr.25177) 33 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:21 smithi082 bash[27462]: audit 2023-10-09T04:15:20.829991+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.82:0/249742919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:22.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:22 smithi153 bash[48582]: ts=2023-10-09T04:15:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.82:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:23.164 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:15:22 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:15:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:23 smithi153 bash[27868]: cluster 2023-10-09T04:15:21.834988+0000 mgr.x (mgr.25177) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:23 smithi082 bash[33901]: cluster 2023-10-09T04:15:21.834988+0000 mgr.x (mgr.25177) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:23.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:23 smithi082 bash[27462]: cluster 2023-10-09T04:15:21.834988+0000 mgr.x (mgr.25177) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:24 smithi153 bash[27868]: audit 2023-10-09T04:15:23.273056+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.82:0/3442348614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:24.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:15:23 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:23] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:15:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:24 smithi082 bash[27462]: audit 2023-10-09T04:15:23.273056+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.82:0/3442348614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:24 smithi082 bash[33901]: audit 2023-10-09T04:15:23.273056+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.82:0/3442348614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:25.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:25 smithi082 bash[27462]: cluster 2023-10-09T04:15:23.836132+0000 mgr.x (mgr.25177) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:25.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:25 smithi082 bash[33901]: cluster 2023-10-09T04:15:23.836132+0000 mgr.x (mgr.25177) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:25 smithi153 bash[27868]: cluster 2023-10-09T04:15:23.836132+0000 mgr.x (mgr.25177) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T04:15:26.343 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:26 smithi082 bash[33901]: audit 2023-10-09T04:15:25.714037+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.82:0/2917983455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:26.343 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:26 smithi082 bash[33901]: audit 2023-10-09T04:15:26.093637+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:26.343 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:26 smithi082 bash[27462]: audit 2023-10-09T04:15:25.714037+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.82:0/2917983455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:26.343 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:26 smithi082 bash[27462]: audit 2023-10-09T04:15:26.093637+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:26 smithi153 bash[27868]: audit 2023-10-09T04:15:25.714037+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.82:0/2917983455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:26 smithi153 bash[27868]: audit 2023-10-09T04:15:26.093637+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:27.188 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:26 smithi082 systemd[1]: Stopping Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:27.189 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:26 smithi082 bash[72192]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T04:15:27.189 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 bash[56808]: level=info ts=2023-10-09T04:15:27.101Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:15:27.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:27 smithi082 bash[33901]: cluster 2023-10-09T04:15:25.836739+0000 mgr.x (mgr.25177) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:27.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:27 smithi082 bash[27462]: cluster 2023-10-09T04:15:25.836739+0000 mgr.x (mgr.25177) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:27.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:27 smithi153 bash[27868]: cluster 2023-10-09T04:15:25.836739+0000 mgr.x (mgr.25177) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:27.955 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 bash[72226]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:15:27.955 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 bash[72309]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T04:15:27.955 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:15:27.955 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 systemd[1]: Stopped Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:27.955 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:27 smithi082 systemd[1]: Started Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.000Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.000Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.004Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.82 port=9094 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.005Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.051Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.051Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.055Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T04:15:29.233 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:29 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:29.055Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: cluster 2023-10-09T04:15:27.837848+0000 mgr.x (mgr.25177) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: audit 2023-10-09T04:15:27.935971+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: audit 2023-10-09T04:15:27.948515+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: cephadm 2023-10-09T04:15:27.979400+0000 mgr.x (mgr.25177) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: audit 2023-10-09T04:15:28.150915+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.82:0/2049625488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[27462]: cephadm 2023-10-09T04:15:28.225630+0000 mgr.x (mgr.25177) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: cluster 2023-10-09T04:15:27.837848+0000 mgr.x (mgr.25177) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:29.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: audit 2023-10-09T04:15:27.935971+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: audit 2023-10-09T04:15:27.948515+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: cephadm 2023-10-09T04:15:27.979400+0000 mgr.x (mgr.25177) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:15:29.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: audit 2023-10-09T04:15:28.150915+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.82:0/2049625488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:29.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:28 smithi082 bash[33901]: cephadm 2023-10-09T04:15:28.225630+0000 mgr.x (mgr.25177) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: cluster 2023-10-09T04:15:27.837848+0000 mgr.x (mgr.25177) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: audit 2023-10-09T04:15:27.935971+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: audit 2023-10-09T04:15:27.948515+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: cephadm 2023-10-09T04:15:27.979400+0000 mgr.x (mgr.25177) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: audit 2023-10-09T04:15:28.150915+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.82:0/2049625488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:29.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:28 smithi153 bash[27868]: cephadm 2023-10-09T04:15:28.225630+0000 mgr.x (mgr.25177) 39 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:15:31.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:31 smithi153 bash[27868]: cluster 2023-10-09T04:15:29.838298+0000 mgr.x (mgr.25177) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:31.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:31 smithi153 bash[27868]: audit 2023-10-09T04:15:30.595526+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.82:0/3392004559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:31.482 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:31 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:31.006Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001028457s 2023-10-09T04:15:31.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:31 smithi082 bash[27462]: cluster 2023-10-09T04:15:29.838298+0000 mgr.x (mgr.25177) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:31.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:31 smithi082 bash[27462]: audit 2023-10-09T04:15:30.595526+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.82:0/3392004559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:31 smithi082 bash[33901]: cluster 2023-10-09T04:15:29.838298+0000 mgr.x (mgr.25177) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:31 smithi082 bash[33901]: audit 2023-10-09T04:15:30.595526+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.82:0/3392004559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:32.676 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:32 smithi153 bash[48582]: ts=2023-10-09T04:15:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.82:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:33.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:33 smithi082 bash[27462]: cluster 2023-10-09T04:15:31.838847+0000 mgr.x (mgr.25177) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:33.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:33 smithi082 bash[27462]: audit 2023-10-09T04:15:33.033024+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.82:0/1152013839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:33.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:15:32 smithi082 bash[27740]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:15:33.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:33 smithi082 bash[33901]: cluster 2023-10-09T04:15:31.838847+0000 mgr.x (mgr.25177) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:33.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:33 smithi082 bash[33901]: audit 2023-10-09T04:15:33.033024+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.82:0/1152013839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:33.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:33 smithi153 bash[27868]: cluster 2023-10-09T04:15:31.838847+0000 mgr.x (mgr.25177) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:33.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:33 smithi153 bash[27868]: audit 2023-10-09T04:15:33.033024+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.82:0/1152013839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:34.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:15:33 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:33] "GET /metrics HTTP/1.1" 200 33779 "" "Prometheus/2.33.4" 2023-10-09T04:15:35.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:35 smithi153 bash[27868]: cluster 2023-10-09T04:15:33.839952+0000 mgr.x (mgr.25177) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:35.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:35 smithi082 bash[27462]: cluster 2023-10-09T04:15:33.839952+0000 mgr.x (mgr.25177) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:35.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:35 smithi082 bash[33901]: cluster 2023-10-09T04:15:33.839952+0000 mgr.x (mgr.25177) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:36.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:36 smithi153 bash[27868]: audit 2023-10-09T04:15:35.480530+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.82:0/1293282149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:36 smithi082 bash[27462]: audit 2023-10-09T04:15:35.480530+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.82:0/1293282149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:36 smithi082 bash[33901]: audit 2023-10-09T04:15:35.480530+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.82:0/1293282149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:37 smithi153 bash[27868]: cluster 2023-10-09T04:15:35.840690+0000 mgr.x (mgr.25177) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:37.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:37 smithi082 bash[27462]: cluster 2023-10-09T04:15:35.840690+0000 mgr.x (mgr.25177) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:37.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:37 smithi082 bash[33901]: cluster 2023-10-09T04:15:35.840690+0000 mgr.x (mgr.25177) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:38.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:38 smithi153 bash[27868]: audit 2023-10-09T04:15:37.921274+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.82:0/1013157880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:38 smithi082 bash[27462]: audit 2023-10-09T04:15:37.921274+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.82:0/1013157880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:38 smithi082 bash[33901]: audit 2023-10-09T04:15:37.921274+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.82:0/1013157880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:39.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[27868]: cluster 2023-10-09T04:15:37.841805+0000 mgr.x (mgr.25177) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:39.426 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 systemd[1]: Stopping Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:39.426 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[53104]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T04:15:39.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:39 smithi082 bash[27462]: cluster 2023-10-09T04:15:37.841805+0000 mgr.x (mgr.25177) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:39 smithi082 bash[33901]: cluster 2023-10-09T04:15:37.841805+0000 mgr.x (mgr.25177) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:39.486 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 04:15:39 smithi082 bash[72338]: level=info ts=2023-10-09T04:15:39.010Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.005023192s 2023-10-09T04:15:39.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:15:39.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:15:39.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:15:39.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T04:15:39.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.553Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.554Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.554Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T04:15:39.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:39 smithi153 bash[48582]: ts=2023-10-09T04:15:39.554Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T04:15:40.366 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:40 smithi153 bash[53142]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:15:40.366 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:40 smithi153 bash[53204]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:15:40.366 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:40 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:15:40.366 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:40 smithi153 systemd[1]: Stopped Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:40.366 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:40 smithi153 systemd[1]: Started Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:40.926 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:40 smithi153 systemd[1]: Stopping Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:40.926 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:40 smithi153 bash[53256]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter.b 2023-10-09T04:15:41.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[27868]: cluster 2023-10-09T04:15:39.842390+0000 mgr.x (mgr.25177) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:41.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[27868]: audit 2023-10-09T04:15:40.346591+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[27868]: audit 2023-10-09T04:15:40.357782+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[27868]: audit 2023-10-09T04:15:40.367565+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.82:0/2099281979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:41.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[27868]: audit 2023-10-09T04:15:41.093428+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[33901]: cluster 2023-10-09T04:15:39.842390+0000 mgr.x (mgr.25177) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[33901]: audit 2023-10-09T04:15:40.346591+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[33901]: audit 2023-10-09T04:15:40.357782+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[33901]: audit 2023-10-09T04:15:40.367565+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.82:0/2099281979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[33901]: audit 2023-10-09T04:15:41.093428+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[27462]: cluster 2023-10-09T04:15:39.842390+0000 mgr.x (mgr.25177) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[27462]: audit 2023-10-09T04:15:40.346591+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[27462]: audit 2023-10-09T04:15:40.357782+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:41.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[27462]: audit 2023-10-09T04:15:40.367565+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.82:0/2099281979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:41.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:41 smithi082 bash[27462]: audit 2023-10-09T04:15:41.093428+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:41.926 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53291]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T04:15:41.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.657Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T04:15:41.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.657Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T04:15:41.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.657Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi153 (none))" 2023-10-09T04:15:41.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.657Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.657Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.659Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.659Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.661Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.665Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.665Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.475µs 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.665Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:15:41.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53233]: ts=2023-10-09T04:15:41.810Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T04:15:42.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[27868]: cephadm 2023-10-09T04:15:40.376387+0000 mgr.x (mgr.25177) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:15:42.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[27868]: cephadm 2023-10-09T04:15:40.377116+0000 mgr.x (mgr.25177) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi153 2023-10-09T04:15:42.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[27868]: audit 2023-10-09T04:15:42.048640+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:42.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[27868]: audit 2023-10-09T04:15:42.065640+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:42.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:41 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:15:42.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:41 smithi153 bash[53425]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T04:15:42.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:42 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T04:15:42.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:42 smithi153 systemd[1]: Stopped Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:42.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:42 smithi153 systemd[1]: Started Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:42.427 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.043Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T04:15:42.427 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.043Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.043Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=76.699µs wal_replay_duration=378.225153ms total_replay_duration=378.322026ms 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.048Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.048Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.048Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.075Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.988668ms db_storage=1.291µs remote_storage=4.017µs web_handler=685ns query_engine=1.059µs scrape=327.017µs scrape_sd=114.942µs notify=12.875µs notify_sd=24.301µs rules=26.217877ms 2023-10-09T04:15:42.428 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:42 smithi153 bash[53233]: ts=2023-10-09T04:15:42.075Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T04:15:42.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[33901]: cephadm 2023-10-09T04:15:40.376387+0000 mgr.x (mgr.25177) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[33901]: cephadm 2023-10-09T04:15:40.377116+0000 mgr.x (mgr.25177) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi153 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[33901]: audit 2023-10-09T04:15:42.048640+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[33901]: audit 2023-10-09T04:15:42.065640+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[27462]: cephadm 2023-10-09T04:15:40.376387+0000 mgr.x (mgr.25177) 46 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[27462]: cephadm 2023-10-09T04:15:40.377116+0000 mgr.x (mgr.25177) 47 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi153 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[27462]: audit 2023-10-09T04:15:42.048640+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:42.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:42 smithi082 bash[27462]: audit 2023-10-09T04:15:42.065640+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:15:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[27868]: cluster 2023-10-09T04:15:41.842992+0000 mgr.x (mgr.25177) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[27868]: cephadm 2023-10-09T04:15:42.095801+0000 mgr.x (mgr.25177) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:15:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[27868]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:15:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[27868]: audit 2023-10-09T04:15:42.806126+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.82:0/637543802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:43.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.236Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T04:15:43.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.236Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T04:15:43.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.237Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T04:15:43.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.237Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:15:43.427 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T04:15:43.428 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T04:15:43.429 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T04:15:43.430 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T04:15:43.431 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T04:15:43.432 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T04:15:43.432 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T04:15:43.432 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T04:15:43.432 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T04:15:43.432 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 04:15:43 smithi153 bash[53454]: ts=2023-10-09T04:15:43.238Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T04:15:43.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[27462]: cluster 2023-10-09T04:15:41.842992+0000 mgr.x (mgr.25177) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:43.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[27462]: cephadm 2023-10-09T04:15:42.095801+0000 mgr.x (mgr.25177) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:15:43.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[27462]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:15:43.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[27462]: audit 2023-10-09T04:15:42.806126+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.82:0/637543802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:43.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[33901]: cluster 2023-10-09T04:15:41.842992+0000 mgr.x (mgr.25177) 48 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:43.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[33901]: cephadm 2023-10-09T04:15:42.095801+0000 mgr.x (mgr.25177) 49 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:15:43.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[33901]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm [INF] Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:15:43.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:43 smithi082 bash[33901]: audit 2023-10-09T04:15:42.806126+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.82:0/637543802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:45 smithi082 bash[27462]: cluster 2023-10-09T04:15:43.844135+0000 mgr.x (mgr.25177) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:45 smithi082 bash[33901]: cluster 2023-10-09T04:15:43.844135+0000 mgr.x (mgr.25177) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:45 smithi153 bash[27868]: cluster 2023-10-09T04:15:43.844135+0000 mgr.x (mgr.25177) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:46 smithi082 bash[33901]: audit 2023-10-09T04:15:45.244730+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.82:0/1101530667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:46 smithi082 bash[27462]: audit 2023-10-09T04:15:45.244730+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.82:0/1101530667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:46 smithi153 bash[27868]: audit 2023-10-09T04:15:45.244730+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.82:0/1101530667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:47 smithi082 bash[27462]: cluster 2023-10-09T04:15:45.844640+0000 mgr.x (mgr.25177) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:47.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:47 smithi082 bash[33901]: cluster 2023-10-09T04:15:45.844640+0000 mgr.x (mgr.25177) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:47 smithi153 bash[27868]: cluster 2023-10-09T04:15:45.844640+0000 mgr.x (mgr.25177) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:48.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:48 smithi082 bash[27462]: audit 2023-10-09T04:15:47.674197+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.82:0/1332238988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:48.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:48 smithi082 bash[33901]: audit 2023-10-09T04:15:47.674197+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.82:0/1332238988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:48 smithi153 bash[27868]: audit 2023-10-09T04:15:47.674197+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.82:0/1332238988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:49 smithi082 bash[27462]: cluster 2023-10-09T04:15:47.845505+0000 mgr.x (mgr.25177) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:49 smithi082 bash[33901]: cluster 2023-10-09T04:15:47.845505+0000 mgr.x (mgr.25177) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:49 smithi153 bash[27868]: cluster 2023-10-09T04:15:47.845505+0000 mgr.x (mgr.25177) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:50 smithi082 bash[27462]: audit 2023-10-09T04:15:50.120629+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.82:0/424515007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:50.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:50 smithi082 bash[33901]: audit 2023-10-09T04:15:50.120629+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.82:0/424515007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:50 smithi153 bash[27868]: audit 2023-10-09T04:15:50.120629+0000 mon.a (mon.0) 872 : audit [DBG] from='client.? 172.21.15.82:0/424515007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:51.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:51 smithi082 bash[27462]: cluster 2023-10-09T04:15:49.846062+0000 mgr.x (mgr.25177) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:51 smithi082 bash[33901]: cluster 2023-10-09T04:15:49.846062+0000 mgr.x (mgr.25177) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:51 smithi153 bash[27868]: cluster 2023-10-09T04:15:49.846062+0000 mgr.x (mgr.25177) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:53.233 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:15:52 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:15:52] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.33.4" 2023-10-09T04:15:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:53 smithi153 bash[27868]: cluster 2023-10-09T04:15:51.846613+0000 mgr.x (mgr.25177) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:53 smithi153 bash[27868]: audit 2023-10-09T04:15:52.562945+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.82:0/1863806341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:53.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:53 smithi082 bash[27462]: cluster 2023-10-09T04:15:51.846613+0000 mgr.x (mgr.25177) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:53.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:53 smithi082 bash[27462]: audit 2023-10-09T04:15:52.562945+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.82:0/1863806341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:53 smithi082 bash[33901]: cluster 2023-10-09T04:15:51.846613+0000 mgr.x (mgr.25177) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:53.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:53 smithi082 bash[33901]: audit 2023-10-09T04:15:52.562945+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.82:0/1863806341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:55 smithi153 bash[27868]: cluster 2023-10-09T04:15:53.847431+0000 mgr.x (mgr.25177) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:55.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:55 smithi153 bash[27868]: audit 2023-10-09T04:15:54.992065+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.82:0/68338931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:55 smithi082 bash[27462]: cluster 2023-10-09T04:15:53.847431+0000 mgr.x (mgr.25177) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:55.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:55 smithi082 bash[27462]: audit 2023-10-09T04:15:54.992065+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.82:0/68338931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:55 smithi082 bash[33901]: cluster 2023-10-09T04:15:53.847431+0000 mgr.x (mgr.25177) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:55.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:55 smithi082 bash[33901]: audit 2023-10-09T04:15:54.992065+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.82:0/68338931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:56 smithi153 bash[27868]: audit 2023-10-09T04:15:56.097818+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:56.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:56 smithi082 bash[27462]: audit 2023-10-09T04:15:56.097818+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:56.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:56 smithi082 bash[33901]: audit 2023-10-09T04:15:56.097818+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:57.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:57 smithi153 bash[27868]: cluster 2023-10-09T04:15:55.847914+0000 mgr.x (mgr.25177) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:57.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:57 smithi082 bash[27462]: cluster 2023-10-09T04:15:55.847914+0000 mgr.x (mgr.25177) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:57.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:57 smithi082 bash[33901]: cluster 2023-10-09T04:15:55.847914+0000 mgr.x (mgr.25177) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:58.661 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:58 smithi153 bash[27868]: audit 2023-10-09T04:15:57.431833+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.82:0/762006238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:58.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:58 smithi082 bash[33901]: audit 2023-10-09T04:15:57.431833+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.82:0/762006238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:58.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:58 smithi082 bash[27462]: audit 2023-10-09T04:15:57.431833+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.82:0/762006238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:58.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:15:58 smithi153 bash[53233]: ts=2023-10-09T04:15:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:15:59 smithi153 bash[27868]: cluster 2023-10-09T04:15:57.849114+0000 mgr.x (mgr.25177) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:15:59 smithi082 bash[27462]: cluster 2023-10-09T04:15:57.849114+0000 mgr.x (mgr.25177) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:15:59.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:15:59 smithi082 bash[33901]: cluster 2023-10-09T04:15:57.849114+0000 mgr.x (mgr.25177) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:00 smithi153 bash[27868]: audit 2023-10-09T04:15:59.871994+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.82:0/2483318457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:00.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:00 smithi082 bash[27462]: audit 2023-10-09T04:15:59.871994+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.82:0/2483318457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:00.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:00 smithi082 bash[33901]: audit 2023-10-09T04:15:59.871994+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.82:0/2483318457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:01.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:01 smithi153 bash[27868]: cluster 2023-10-09T04:15:59.849779+0000 mgr.x (mgr.25177) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:01.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:01 smithi082 bash[27462]: cluster 2023-10-09T04:15:59.849779+0000 mgr.x (mgr.25177) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:01.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:01 smithi082 bash[33901]: cluster 2023-10-09T04:15:59.849779+0000 mgr.x (mgr.25177) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:03.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:02 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:16:02] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-09T04:16:03.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:03 smithi153 bash[27868]: cluster 2023-10-09T04:16:01.850492+0000 mgr.x (mgr.25177) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:03.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:03 smithi153 bash[27868]: audit 2023-10-09T04:16:02.309716+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.82:0/1638674991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:03 smithi082 bash[27462]: cluster 2023-10-09T04:16:01.850492+0000 mgr.x (mgr.25177) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:03.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:03 smithi082 bash[27462]: audit 2023-10-09T04:16:02.309716+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.82:0/1638674991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:03.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:03 smithi082 bash[33901]: cluster 2023-10-09T04:16:01.850492+0000 mgr.x (mgr.25177) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:03.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:03 smithi082 bash[33901]: audit 2023-10-09T04:16:02.309716+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.82:0/1638674991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:04.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:04 smithi153 systemd[1]: Stopping Ceph grafana.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:16:04.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:04 smithi153 bash[53716]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-grafana.a 2023-10-09T04:16:04.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:04 smithi153 bash[47868]: t=2023-10-09T04:16:04+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T04:16:04.676 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:04 smithi153 bash[47868]: t=2023-10-09T04:16:04+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-10-09T04:16:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:05 smithi153 bash[27868]: cluster 2023-10-09T04:16:03.851332+0000 mgr.x (mgr.25177) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:05 smithi153 bash[27868]: audit 2023-10-09T04:16:04.751012+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.82:0/3656750505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:05.426 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:05 smithi153 bash[53750]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T04:16:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:05 smithi082 bash[27462]: cluster 2023-10-09T04:16:03.851332+0000 mgr.x (mgr.25177) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:05 smithi082 bash[27462]: audit 2023-10-09T04:16:04.751012+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.82:0/3656750505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:05 smithi082 bash[33901]: cluster 2023-10-09T04:16:03.851332+0000 mgr.x (mgr.25177) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:05 smithi082 bash[33901]: audit 2023-10-09T04:16:04.751012+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.82:0/3656750505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:05.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:05 smithi153 bash[53812]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T04:16:05.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:05 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T04:16:05.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:05 smithi153 systemd[1]: Stopped Ceph grafana.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:05.926 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:05 smithi153 systemd[1]: Started Ceph grafana.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.487339+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.493830+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.535542+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.536866+0000 mgr.x (mgr.25177) 62 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.541206+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.541695+0000 mgr.x (mgr.25177) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.545396+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.546084+0000 mgr.x (mgr.25177) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: audit 2023-10-09T04:16:05.557977+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:06.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[27868]: cluster 2023-10-09T04:16:05.851934+0000 mgr.x (mgr.25177) 67 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:06.679 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T04:16:06.679 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T04:16:06.679 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T04:16:06.680 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T04:16:06.680 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T04:16:06.680 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T04:16:06.680 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T04:16:06.680 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T04:16:06.681 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=649.96µs 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T04:16:06.682 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T04:16:06.683 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T04:16:06.683 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T04:16:06.683 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T04:16:06.683 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.487339+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.493830+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.535542+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.536866+0000 mgr.x (mgr.25177) 62 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.541206+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.541695+0000 mgr.x (mgr.25177) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.545396+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.546084+0000 mgr.x (mgr.25177) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: audit 2023-10-09T04:16:05.557977+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[27462]: cluster 2023-10-09T04:16:05.851934+0000 mgr.x (mgr.25177) 67 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.487339+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.493830+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.535542+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.536866+0000 mgr.x (mgr.25177) 62 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:16:06.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.541206+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.541695+0000 mgr.x (mgr.25177) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.545396+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.546084+0000 mgr.x (mgr.25177) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: audit 2023-10-09T04:16:05.557977+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:06.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:06 smithi082 bash[33901]: cluster 2023-10-09T04:16:05.851934+0000 mgr.x (mgr.25177) 67 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:07.176 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T04:16:07.176 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:16:07.176 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T04:16:07.176 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T04:16:07.177 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:16:07.177 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 04:16:06 smithi153 bash[53843]: t=2023-10-09T04:16:06+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T04:16:07.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:07 smithi153 bash[27868]: audit 2023-10-09T04:16:07.202583+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.82:0/1734428506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:07.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:07 smithi082 bash[27462]: audit 2023-10-09T04:16:07.202583+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.82:0/1734428506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:07.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:07 smithi082 bash[33901]: audit 2023-10-09T04:16:07.202583+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.82:0/1734428506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:08.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[53233]: ts=2023-10-09T04:16:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: audit 2023-10-09T04:16:07.690805+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: audit 2023-10-09T04:16:07.693193+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: audit 2023-10-09T04:16:07.693408+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: audit 2023-10-09T04:16:07.695882+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: audit 2023-10-09T04:16:07.697304+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm [INF] Deploying daemon mgr.y on smithi082 2023-10-09T04:16:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:08 smithi153 bash[27868]: cluster 2023-10-09T04:16:07.852935+0000 mgr.x (mgr.25177) 70 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: audit 2023-10-09T04:16:07.690805+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: audit 2023-10-09T04:16:07.693193+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: audit 2023-10-09T04:16:07.693408+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: audit 2023-10-09T04:16:07.695882+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: audit 2023-10-09T04:16:07.697304+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm [INF] Deploying daemon mgr.y on smithi082 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[27462]: cluster 2023-10-09T04:16:07.852935+0000 mgr.x (mgr.25177) 70 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T04:16:08.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: audit 2023-10-09T04:16:07.690805+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: audit 2023-10-09T04:16:07.693193+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: audit 2023-10-09T04:16:07.693408+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: audit 2023-10-09T04:16:07.695882+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: audit 2023-10-09T04:16:07.697304+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm [INF] Deploying daemon mgr.y on smithi082 2023-10-09T04:16:08.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:08 smithi082 bash[33901]: cluster 2023-10-09T04:16:07.852935+0000 mgr.x (mgr.25177) 70 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:10.034 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 systemd[1]: Stopping Ceph mgr.y for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:16:10.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:10 smithi082 bash[27462]: audit 2023-10-09T04:16:09.631436+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.82:0/1716076792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:10.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:10 smithi082 bash[33901]: audit 2023-10-09T04:16:09.631436+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.82:0/1716076792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:10 smithi153 bash[27868]: audit 2023-10-09T04:16:09.631436+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.82:0/1716076792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:10.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 bash[73077]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T04:16:10.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:16:10.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 bash[73140]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T04:16:10.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T04:16:10.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:10 smithi082 systemd[1]: Stopped Ceph mgr.y for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:11 smithi153 bash[27868]: cluster 2023-10-09T04:16:09.853540+0000 mgr.x (mgr.25177) 71 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:11 smithi153 bash[27868]: audit 2023-10-09T04:16:11.098351+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:11.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:11 smithi082 bash[27462]: cluster 2023-10-09T04:16:09.853540+0000 mgr.x (mgr.25177) 71 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:11.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:11 smithi082 bash[27462]: audit 2023-10-09T04:16:11.098351+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:11.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:11 smithi082 systemd[1]: Started Ceph mgr.y for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:11.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:11 smithi082 bash[33901]: cluster 2023-10-09T04:16:09.853540+0000 mgr.x (mgr.25177) 71 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:11.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:11 smithi082 bash[33901]: audit 2023-10-09T04:16:11.098351+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:12.340 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[73210]: debug 2023-10-09T04:16:12.172+0000 7fe1e7d22700 1 -- 172.21.15.82:0/2355285431 <== mon.0 v2:172.21.15.82:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e840d5d1e0 con 0x55e841adcc00 2023-10-09T04:16:12.340 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[33901]: audit 2023-10-09T04:16:11.186679+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.340 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[33901]: audit 2023-10-09T04:16:11.197262+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.340 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[33901]: audit 2023-10-09T04:16:11.223618+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:12.340 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[33901]: audit 2023-10-09T04:16:12.069842+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.82:0/1982580148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:12.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[27462]: audit 2023-10-09T04:16:11.186679+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[27462]: audit 2023-10-09T04:16:11.197262+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[27462]: audit 2023-10-09T04:16:11.223618+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:12.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[27462]: audit 2023-10-09T04:16:12.069842+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.82:0/1982580148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:12 smithi153 bash[27868]: audit 2023-10-09T04:16:11.186679+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:12 smithi153 bash[27868]: audit 2023-10-09T04:16:11.197262+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:12 smithi153 bash[27868]: audit 2023-10-09T04:16:11.223618+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:12 smithi153 bash[27868]: audit 2023-10-09T04:16:12.069842+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.82:0/1982580148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:12.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:12 smithi082 bash[73210]: debug 2023-10-09T04:16:12.336+0000 7fe1f1e70200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:16:13.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:13 smithi153 bash[27868]: cluster 2023-10-09T04:16:11.854133+0000 mgr.x (mgr.25177) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:13.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:12 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:16:12] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-09T04:16:13.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[27462]: cluster 2023-10-09T04:16:11.854133+0000 mgr.x (mgr.25177) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:13.469 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[73210]: debug 2023-10-09T04:16:13.256+0000 7fe1f1e70200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:16:13.469 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[73210]: debug 2023-10-09T04:16:13.360+0000 7fe1f1e70200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:16:13.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[33901]: cluster 2023-10-09T04:16:11.854133+0000 mgr.x (mgr.25177) 72 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:13.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[73210]: debug 2023-10-09T04:16:13.464+0000 7fe1f1e70200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:16:14.102 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:13 smithi082 bash[73210]: debug 2023-10-09T04:16:13.732+0000 7fe1f1e70200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:16:14.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:14 smithi082 bash[73210]: debug 2023-10-09T04:16:14.096+0000 7fe1f1e70200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:16:14.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:14 smithi082 bash[73210]: debug 2023-10-09T04:16:14.268+0000 7fe1f1e70200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:16:14.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:14 smithi082 bash[73210]: debug 2023-10-09T04:16:14.488+0000 7fe1f1e70200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:16:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[27462]: cluster 2023-10-09T04:16:13.855259+0000 mgr.x (mgr.25177) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:15.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[27462]: audit 2023-10-09T04:16:14.511643+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.82:0/3962555194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[33901]: cluster 2023-10-09T04:16:13.855259+0000 mgr.x (mgr.25177) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[33901]: audit 2023-10-09T04:16:14.511643+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.82:0/3962555194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:15 smithi153 bash[27868]: cluster 2023-10-09T04:16:13.855259+0000 mgr.x (mgr.25177) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:15 smithi153 bash[27868]: audit 2023-10-09T04:16:14.511643+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.82:0/3962555194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:15.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[73210]: debug 2023-10-09T04:16:15.536+0000 7fe1f1e70200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:16:15.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[73210]: debug 2023-10-09T04:16:15.656+0000 7fe1f1e70200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:16:15.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:15 smithi082 bash[73210]: debug 2023-10-09T04:16:15.776+0000 7fe1f1e70200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:16:16.548 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:16 smithi082 bash[73210]: debug 2023-10-09T04:16:16.288+0000 7fe1f1e70200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:16:16.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:16 smithi082 bash[73210]: debug 2023-10-09T04:16:16.544+0000 7fe1f1e70200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[27462]: cluster 2023-10-09T04:16:15.855902+0000 mgr.x (mgr.25177) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[27462]: audit 2023-10-09T04:16:16.977222+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.82:0/3175561686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[27462]: audit 2023-10-09T04:16:17.172177+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[27462]: audit 2023-10-09T04:16:17.181728+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[73210]: debug 2023-10-09T04:16:17.068+0000 7fe1f1e70200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:16:17.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[33901]: cluster 2023-10-09T04:16:15.855902+0000 mgr.x (mgr.25177) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:17.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[33901]: audit 2023-10-09T04:16:16.977222+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.82:0/3175561686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:17.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[33901]: audit 2023-10-09T04:16:17.172177+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[33901]: audit 2023-10-09T04:16:17.181728+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:17 smithi153 bash[27868]: cluster 2023-10-09T04:16:15.855902+0000 mgr.x (mgr.25177) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:17 smithi153 bash[27868]: audit 2023-10-09T04:16:16.977222+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.82:0/3175561686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:17 smithi153 bash[27868]: audit 2023-10-09T04:16:17.172177+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:17 smithi153 bash[27868]: audit 2023-10-09T04:16:17.181728+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:17.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:17 smithi082 bash[73210]: debug 2023-10-09T04:16:17.628+0000 7fe1f1e70200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:16:18.533 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[73210]: debug 2023-10-09T04:16:18.272+0000 7fe1f1e70200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:16:18.533 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[73210]: debug 2023-10-09T04:16:18.400+0000 7fe1f1e70200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[27462]: audit 2023-10-09T04:16:17.531221+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[27462]: audit 2023-10-09T04:16:17.540239+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[27462]: cluster 2023-10-09T04:16:17.856817+0000 mgr.x (mgr.25177) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[27462]: audit 2023-10-09T04:16:18.034084+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[27462]: audit 2023-10-09T04:16:18.043126+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[33901]: audit 2023-10-09T04:16:17.531221+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[33901]: audit 2023-10-09T04:16:17.540239+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[33901]: cluster 2023-10-09T04:16:17.856817+0000 mgr.x (mgr.25177) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:18.787 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[33901]: audit 2023-10-09T04:16:18.034084+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.788 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[33901]: audit 2023-10-09T04:16:18.043126+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[27868]: audit 2023-10-09T04:16:17.531221+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[27868]: audit 2023-10-09T04:16:17.540239+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[27868]: cluster 2023-10-09T04:16:17.856817+0000 mgr.x (mgr.25177) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[27868]: audit 2023-10-09T04:16:18.034084+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[27868]: audit 2023-10-09T04:16:18.043126+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:18.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:18 smithi153 bash[53233]: ts=2023-10-09T04:16:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:19.060 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[73210]: debug 2023-10-09T04:16:18.780+0000 7fe1f1e70200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:16:19.060 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:18 smithi082 bash[73210]: debug 2023-10-09T04:16:18.900+0000 7fe1f1e70200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:16:19.060 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:19 smithi082 bash[73210]: debug 2023-10-09T04:16:19.056+0000 7fe1f1e70200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:16:19.316 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:19 smithi082 bash[73210]: debug 2023-10-09T04:16:19.180+0000 7fe1f1e70200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:16:19.648 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:19 smithi082 bash[73210]: debug 2023-10-09T04:16:19.312+0000 7fe1f1e70200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:16:19.649 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:19 smithi082 bash[73210]: debug 2023-10-09T04:16:19.512+0000 7fe1f1e70200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:16:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:19 smithi082 bash[73210]: debug 2023-10-09T04:16:19.644+0000 7fe1f1e70200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:16:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:20 smithi153 bash[27868]: audit 2023-10-09T04:16:19.420562+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.82:0/3758127426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:20 smithi082 bash[27462]: audit 2023-10-09T04:16:19.420562+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.82:0/3758127426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:20 smithi082 bash[33901]: audit 2023-10-09T04:16:19.420562+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.82:0/3758127426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:20 smithi082 bash[73210]: debug 2023-10-09T04:16:20.900+0000 7fe1f1e70200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:20 smithi082 bash[73210]: [09/Oct/2023:04:16:20] ENGINE Bus STARTING 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[73210]: CherryPy Checker: 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[73210]: The Application mounted at '' has an empty config. 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[73210]: [09/Oct/2023:04:16:21] ENGINE Serving on http://:::9283 2023-10-09T04:16:21.157 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[73210]: [09/Oct/2023:04:16:21] ENGINE Bus STARTED 2023-10-09T04:16:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: cluster 2023-10-09T04:16:19.857402+0000 mgr.x (mgr.25177) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: cluster 2023-10-09T04:16:20.927637+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: cluster 2023-10-09T04:16:20.927921+0000 mon.a (mon.0) 890 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: audit 2023-10-09T04:16:20.935200+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: audit 2023-10-09T04:16:20.936506+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: audit 2023-10-09T04:16:20.941612+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:21 smithi153 bash[27868]: audit 2023-10-09T04:16:20.942218+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: cluster 2023-10-09T04:16:19.857402+0000 mgr.x (mgr.25177) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: cluster 2023-10-09T04:16:20.927637+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:16:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: cluster 2023-10-09T04:16:20.927921+0000 mon.a (mon.0) 890 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:16:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: audit 2023-10-09T04:16:20.935200+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:16:21.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: audit 2023-10-09T04:16:20.936506+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: audit 2023-10-09T04:16:20.941612+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[27462]: audit 2023-10-09T04:16:20.942218+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: cluster 2023-10-09T04:16:19.857402+0000 mgr.x (mgr.25177) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: cluster 2023-10-09T04:16:20.927637+0000 mon.a (mon.0) 889 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: cluster 2023-10-09T04:16:20.927921+0000 mon.a (mon.0) 890 : cluster [DBG] Standby manager daemon y started 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: audit 2023-10-09T04:16:20.935200+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: audit 2023-10-09T04:16:20.936506+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: audit 2023-10-09T04:16:20.941612+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:16:21.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:21 smithi082 bash[33901]: audit 2023-10-09T04:16:20.942218+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.? 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:22.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:22 smithi082 bash[27462]: cluster 2023-10-09T04:16:21.173190+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e24: x(active, since 85s), standbys: y 2023-10-09T04:16:22.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:22 smithi082 bash[27462]: audit 2023-10-09T04:16:21.871134+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.82:0/1612250305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:22.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:22 smithi082 bash[33901]: cluster 2023-10-09T04:16:21.173190+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e24: x(active, since 85s), standbys: y 2023-10-09T04:16:22.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:22 smithi082 bash[33901]: audit 2023-10-09T04:16:21.871134+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.82:0/1612250305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:22 smithi153 bash[27868]: cluster 2023-10-09T04:16:21.173190+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e24: x(active, since 85s), standbys: y 2023-10-09T04:16:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:22 smithi153 bash[27868]: audit 2023-10-09T04:16:21.871134+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.82:0/1612250305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:23 smithi153 bash[27868]: cluster 2023-10-09T04:16:21.858030+0000 mgr.x (mgr.25177) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:23.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:22 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:16:22] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-09T04:16:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:23 smithi082 bash[27462]: cluster 2023-10-09T04:16:21.858030+0000 mgr.x (mgr.25177) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:23 smithi082 bash[33901]: cluster 2023-10-09T04:16:21.858030+0000 mgr.x (mgr.25177) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:25 smithi082 bash[27462]: cluster 2023-10-09T04:16:23.859101+0000 mgr.x (mgr.25177) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:25.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:25 smithi082 bash[27462]: audit 2023-10-09T04:16:24.319830+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.82:0/3609369956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:25.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:25 smithi082 bash[33901]: cluster 2023-10-09T04:16:23.859101+0000 mgr.x (mgr.25177) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:25.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:25 smithi082 bash[33901]: audit 2023-10-09T04:16:24.319830+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.82:0/3609369956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:25 smithi153 bash[27868]: cluster 2023-10-09T04:16:23.859101+0000 mgr.x (mgr.25177) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:25 smithi153 bash[27868]: audit 2023-10-09T04:16:24.319830+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.82:0/3609369956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:26 smithi082 bash[33901]: audit 2023-10-09T04:16:26.098640+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:26.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:26 smithi082 bash[27462]: audit 2023-10-09T04:16:26.098640+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:26 smithi153 bash[27868]: audit 2023-10-09T04:16:26.098640+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:27.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE Bus STOPPING 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE Bus STOPPED 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE Bus STARTING 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: cluster 2023-10-09T04:16:25.859718+0000 mgr.x (mgr.25177) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:26.756284+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.82:0/271972363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:27.144956+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:27.153518+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:27.157327+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:27.158735+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[33901]: audit 2023-10-09T04:16:27.166381+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: cluster 2023-10-09T04:16:25.859718+0000 mgr.x (mgr.25177) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:26.756284+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.82:0/271972363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:27.144956+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:27.153518+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:27.157327+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:27.158735+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:27.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[27462]: audit 2023-10-09T04:16:27.166381+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[50739]: debug 2023-10-09T04:16:27.274+0000 7f540b53c700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[50739]: ignoring --setuser ceph since I am not root 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[50739]: ignoring --setgroup ceph since I am not root 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[50739]: debug 2023-10-09T04:16:27.338+0000 7f9aa7b08700 1 -- 172.21.15.153:0/3539939066 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562749f9f1e0 con 0x56274ad1e400 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: cluster 2023-10-09T04:16:25.859718+0000 mgr.x (mgr.25177) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:27.489 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:26.756284+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.82:0/271972363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:27.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:27.144956+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:27.153518+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:27.157327+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:27.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:27.158735+0000 mon.b (mon.1) 95 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:27.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[27868]: audit 2023-10-09T04:16:27.166381+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25177 ' entity='mgr.x' 2023-10-09T04:16:27.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:27 smithi153 bash[50739]: debug 2023-10-09T04:16:27.510+0000 7f9ab1838200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.488+0000 7fe14c2c4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.488+0000 7fe14c2c4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.488+0000 7fe14c2c4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.488+0000 7fe14c2c4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.488+0000 7fe14c2c4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.492+0000 7fe14f2ca700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.492+0000 7fe14f2ca700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.492+0000 7fe14f2ca700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.492+0000 7fe14f2ca700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.984 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: debug 2023-10-09T04:16:27.492+0000 7fe14f2ca700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:16:27.984 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE Serving on http://:::9283 2023-10-09T04:16:27.984 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:27 smithi082 bash[73210]: [09/Oct/2023:04:16:27] ENGINE Bus STARTED 2023-10-09T04:16:28.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cluster 2023-10-09T04:16:27.209325+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T04:16:28.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.253274+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm [INF] Failing over to other MGR 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.265004+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.265413+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cluster 2023-10-09T04:16:27.274201+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.274554+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cluster 2023-10-09T04:16:27.274660+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00850087s) 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.278104+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.278351+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:16:28.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.278535+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.278885+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.279117+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.279662+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.280006+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.280226+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.280440+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:16:28.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.280675+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cluster 2023-10-09T04:16:27.209325+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.253274+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm [INF] Failing over to other MGR 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.265004+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.265413+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cluster 2023-10-09T04:16:27.274201+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.274554+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:16:28.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cluster 2023-10-09T04:16:27.274660+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00850087s) 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.278104+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.278351+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.278535+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.278885+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.279117+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.279662+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.280006+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.280226+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.280440+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.280675+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:16:28.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.280870+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.281090+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.281298+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.281547+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.281916+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.282220+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.282689+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cluster 2023-10-09T04:16:27.328473+0000 mon.a (mon.0) 923 : cluster [INF] Manager daemon y is now available 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.372631+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.378124+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.405766+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:28.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.413811+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: audit 2023-10-09T04:16:27.460023+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[33901]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.280870+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.281090+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.281298+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.281547+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.281916+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.282220+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:16:28.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.282689+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cluster 2023-10-09T04:16:27.328473+0000 mon.a (mon.0) 923 : cluster [INF] Manager daemon y is now available 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.372631+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.378124+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.405766+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.413811+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: audit 2023-10-09T04:16:27.460023+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:16:28.490 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:28 smithi082 bash[27462]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cluster 2023-10-09T04:16:27.209325+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.253274+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm [INF] Failing over to other MGR 2023-10-09T04:16:28.512 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.265004+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25177 172.21.15.153:0/1653265157' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.265413+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cluster 2023-10-09T04:16:27.274201+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.274554+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25177 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cluster 2023-10-09T04:16:27.274660+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00850087s) 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.278104+0000 mon.a (mon.0) 906 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.278351+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.278535+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.278885+0000 mon.a (mon.0) 909 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.279117+0000 mon.a (mon.0) 910 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.279662+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:16:28.513 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.280006+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.280226+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.280440+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.280675+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.280870+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.281090+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.281298+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.281547+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.281916+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.282220+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:16:28.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.282689+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cluster 2023-10-09T04:16:27.328473+0000 mon.a (mon.0) 923 : cluster [INF] Manager daemon y is now available 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.372631+0000 mon.a (mon.0) 924 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.378124+0000 mon.a (mon.0) 925 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.405766+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.413811+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: audit 2023-10-09T04:16:27.460023+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:16:28.515 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[27868]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:16:28.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[53233]: ts=2023-10-09T04:16:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:28.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[50739]: debug 2023-10-09T04:16:28.506+0000 7f9ab1838200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:16:28.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[50739]: debug 2023-10-09T04:16:28.618+0000 7f9ab1838200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:16:28.927 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:28 smithi153 bash[50739]: debug 2023-10-09T04:16:28.730+0000 7f9ab1838200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:16:29.286 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[50739]: debug 2023-10-09T04:16:29.026+0000 7f9ab1838200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:16:29.612 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[50739]: debug 2023-10-09T04:16:29.430+0000 7f9ab1838200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:16:29.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[27868]: cluster 2023-10-09T04:16:28.281753+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e27: y(active, since 1.01559s) 2023-10-09T04:16:29.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[27868]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:16:29.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[27868]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:16:29.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[27868]: cluster 2023-10-09T04:16:28.303795+0000 mgr.y (mgr.15897) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:29.613 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[27868]: audit 2023-10-09T04:16:29.203488+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.82:0/1020234264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:29.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[27462]: cluster 2023-10-09T04:16:28.281753+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e27: y(active, since 1.01559s) 2023-10-09T04:16:29.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[27462]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:16:29.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[27462]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:16:29.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[27462]: cluster 2023-10-09T04:16:28.303795+0000 mgr.y (mgr.15897) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:29.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[27462]: audit 2023-10-09T04:16:29.203488+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.82:0/1020234264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:29.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[33901]: cluster 2023-10-09T04:16:28.281753+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e27: y(active, since 1.01559s) 2023-10-09T04:16:29.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[33901]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:16:29.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[33901]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm [INF] [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:16:29.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[33901]: cluster 2023-10-09T04:16:28.303795+0000 mgr.y (mgr.15897) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:29.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:29 smithi082 bash[33901]: audit 2023-10-09T04:16:29.203488+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.82:0/1020234264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:29.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[50739]: debug 2023-10-09T04:16:29.606+0000 7f9ab1838200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:16:29.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:29 smithi153 bash[50739]: debug 2023-10-09T04:16:29.842+0000 7f9ab1838200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:16:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:30 smithi153 bash[27868]: cluster 2023-10-09T04:16:29.281863+0000 mgr.y (mgr.15897) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:30 smithi153 bash[27868]: cluster 2023-10-09T04:16:29.289007+0000 mon.a (mon.0) 931 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T04:16:30.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:30 smithi082 bash[27462]: cluster 2023-10-09T04:16:29.281863+0000 mgr.y (mgr.15897) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:30.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:30 smithi082 bash[27462]: cluster 2023-10-09T04:16:29.289007+0000 mon.a (mon.0) 931 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T04:16:30.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:30 smithi082 bash[33901]: cluster 2023-10-09T04:16:29.281863+0000 mgr.y (mgr.15897) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:30.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:30 smithi082 bash[33901]: cluster 2023-10-09T04:16:29.289007+0000 mon.a (mon.0) 931 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T04:16:31.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:30 smithi153 bash[50739]: debug 2023-10-09T04:16:30.898+0000 7f9ab1838200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:16:31.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:31 smithi153 bash[50739]: debug 2023-10-09T04:16:31.014+0000 7f9ab1838200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:16:31.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:31 smithi153 bash[50739]: debug 2023-10-09T04:16:31.130+0000 7f9ab1838200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:16:31.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:31 smithi153 bash[50739]: debug 2023-10-09T04:16:31.622+0000 7f9ab1838200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:16:31.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:31 smithi153 bash[50739]: debug 2023-10-09T04:16:31.850+0000 7f9ab1838200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:16:32.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:32 smithi153 bash[50739]: debug 2023-10-09T04:16:32.314+0000 7f9ab1838200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:16:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:32 smithi153 bash[27868]: cluster 2023-10-09T04:16:31.282543+0000 mgr.y (mgr.15897) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:32 smithi153 bash[27868]: audit 2023-10-09T04:16:31.650893+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.82:0/1141770091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:32.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:32 smithi082 bash[27462]: cluster 2023-10-09T04:16:31.282543+0000 mgr.y (mgr.15897) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:32.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:32 smithi082 bash[27462]: audit 2023-10-09T04:16:31.650893+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.82:0/1141770091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:32 smithi082 bash[33901]: cluster 2023-10-09T04:16:31.282543+0000 mgr.y (mgr.15897) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:32 smithi082 bash[33901]: audit 2023-10-09T04:16:31.650893+0000 mon.a (mon.0) 932 : audit [DBG] from='client.? 172.21.15.82:0/1141770091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:33.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:32 smithi153 bash[50739]: debug 2023-10-09T04:16:32.814+0000 7f9ab1838200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:16:33.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:33 smithi153 bash[50739]: debug 2023-10-09T04:16:33.410+0000 7f9ab1838200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:16:33.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:33 smithi153 bash[50739]: debug 2023-10-09T04:16:33.534+0000 7f9ab1838200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:16:34.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:33 smithi153 bash[50739]: debug 2023-10-09T04:16:33.910+0000 7f9ab1838200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:16:34.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.042+0000 7f9ab1838200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:16:34.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.210+0000 7f9ab1838200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:16:34.489 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.338+0000 7f9ab1838200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:16:34.864 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.482+0000 7f9ab1838200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:16:34.864 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.702+0000 7f9ab1838200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:16:35.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[50739]: debug 2023-10-09T04:16:34.858+0000 7f9ab1838200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:16:35.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: cluster 2023-10-09T04:16:33.282952+0000 mgr.y (mgr.15897) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:35.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:33.867184+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:33.876446+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.013809+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.022740+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.106808+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.82:0/2442680309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.673027+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.679798+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.824683+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:34 smithi153 bash[27868]: audit 2023-10-09T04:16:34.830479+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: cluster 2023-10-09T04:16:33.282952+0000 mgr.y (mgr.15897) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:35.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:33.867184+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:33.876446+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.013809+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.022740+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.106808+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.82:0/2442680309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.673027+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.679798+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.824683+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[27462]: audit 2023-10-09T04:16:34.830479+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: cluster 2023-10-09T04:16:33.282952+0000 mgr.y (mgr.15897) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:33.867184+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:33.876446+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.013809+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.022740+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.106808+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.82:0/2442680309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.673027+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.679798+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.824683+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:35.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:34 smithi082 bash[33901]: audit 2023-10-09T04:16:34.830479+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:36.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: debug 2023-10-09T04:16:36.162+0000 7f9ab1838200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:16:36.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: [09/Oct/2023:04:16:36] ENGINE Bus STARTING 2023-10-09T04:16:36.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: CherryPy Checker: 2023-10-09T04:16:36.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: The Application mounted at '' has an empty config. 2023-10-09T04:16:36.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: [09/Oct/2023:04:16:36] ENGINE Serving on http://:::9283 2023-10-09T04:16:36.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[50739]: [09/Oct/2023:04:16:36] ENGINE Bus STARTED 2023-10-09T04:16:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: cluster 2023-10-09T04:16:35.283937+0000 mgr.y (mgr.15897) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: cluster 2023-10-09T04:16:36.187657+0000 mon.a (mon.0) 941 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:16:37.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: audit 2023-10-09T04:16:36.191302+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:16:37.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: audit 2023-10-09T04:16:36.192824+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:37.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: audit 2023-10-09T04:16:36.197094+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:16:37.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: audit 2023-10-09T04:16:36.197593+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:37.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:36 smithi153 bash[27868]: audit 2023-10-09T04:16:36.553595+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.82:0/3803940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: cluster 2023-10-09T04:16:35.283937+0000 mgr.y (mgr.15897) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: cluster 2023-10-09T04:16:36.187657+0000 mon.a (mon.0) 941 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: audit 2023-10-09T04:16:36.191302+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: audit 2023-10-09T04:16:36.192824+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: audit 2023-10-09T04:16:36.197094+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: audit 2023-10-09T04:16:36.197593+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:37.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[27462]: audit 2023-10-09T04:16:36.553595+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.82:0/3803940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: cluster 2023-10-09T04:16:35.283937+0000 mgr.y (mgr.15897) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: cluster 2023-10-09T04:16:36.187657+0000 mon.a (mon.0) 941 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: audit 2023-10-09T04:16:36.191302+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: audit 2023-10-09T04:16:36.192824+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: audit 2023-10-09T04:16:36.197094+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: audit 2023-10-09T04:16:36.197593+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.153:0/4160192270' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:16:37.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:36 smithi082 bash[33901]: audit 2023-10-09T04:16:36.553595+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.82:0/3803940812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:37 smithi153 bash[27868]: cluster 2023-10-09T04:16:36.888247+0000 mon.a (mon.0) 942 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T04:16:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:37 smithi153 bash[27868]: audit 2023-10-09T04:16:36.888704+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:16:38.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:37 smithi082 bash[27462]: cluster 2023-10-09T04:16:36.888247+0000 mon.a (mon.0) 942 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T04:16:38.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:37 smithi082 bash[27462]: audit 2023-10-09T04:16:36.888704+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:16:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:37 smithi082 bash[33901]: cluster 2023-10-09T04:16:36.888247+0000 mon.a (mon.0) 942 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T04:16:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:37 smithi082 bash[33901]: audit 2023-10-09T04:16:36.888704+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:16:39.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:38 smithi153 bash[27868]: cluster 2023-10-09T04:16:37.284577+0000 mgr.y (mgr.15897) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:39.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:38 smithi082 bash[27462]: cluster 2023-10-09T04:16:37.284577+0000 mgr.y (mgr.15897) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:39.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:38 smithi082 bash[33901]: cluster 2023-10-09T04:16:37.284577+0000 mgr.y (mgr.15897) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:40.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:39 smithi153 bash[27868]: audit 2023-10-09T04:16:39.004045+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.82:0/3887914544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:40.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:39 smithi082 bash[27462]: audit 2023-10-09T04:16:39.004045+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.82:0/3887914544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:40.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:39 smithi082 bash[33901]: audit 2023-10-09T04:16:39.004045+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.82:0/3887914544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:41.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:41 smithi153 bash[27868]: cluster 2023-10-09T04:16:39.285687+0000 mgr.y (mgr.15897) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:41 smithi082 bash[33901]: cluster 2023-10-09T04:16:39.285687+0000 mgr.y (mgr.15897) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:41.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:41 smithi082 bash[27462]: cluster 2023-10-09T04:16:39.285687+0000 mgr.y (mgr.15897) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:42.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[27868]: audit 2023-10-09T04:16:41.445442+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.82:0/2438270510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:42.426 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[53233]: ts=2023-10-09T04:16:42.054Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.153:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.153:8765: connect: connection refused" 2023-10-09T04:16:42.426 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[53233]: ts=2023-10-09T04:16:42.058Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.153:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.153:8765: connect: connection refused" 2023-10-09T04:16:42.426 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[53233]: ts=2023-10-09T04:16:42.059Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.153:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.153:8765: connect: connection refused" 2023-10-09T04:16:42.427 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[53233]: ts=2023-10-09T04:16:42.060Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.153:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.153:8765: connect: connection refused" 2023-10-09T04:16:42.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:42 smithi082 bash[27462]: audit 2023-10-09T04:16:41.445442+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.82:0/2438270510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:42.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:42 smithi082 bash[33901]: audit 2023-10-09T04:16:41.445442+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.82:0/2438270510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:43 smithi153 bash[27868]: cluster 2023-10-09T04:16:41.286148+0000 mgr.y (mgr.15897) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:43.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:43 smithi153 bash[27868]: audit 2023-10-09T04:16:42.406382+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:43.427 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:42 smithi153 bash[50739]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:16:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T04:16:43.458 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:43 smithi082 bash[27462]: cluster 2023-10-09T04:16:41.286148+0000 mgr.y (mgr.15897) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:43.458 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:43 smithi082 bash[27462]: audit 2023-10-09T04:16:42.406382+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:43 smithi082 bash[33901]: cluster 2023-10-09T04:16:41.286148+0000 mgr.y (mgr.15897) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:43 smithi082 bash[33901]: audit 2023-10-09T04:16:42.406382+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:44 smithi153 bash[27868]: audit 2023-10-09T04:16:43.877108+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.82:0/4248736776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:44.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:44 smithi153 bash[27868]: audit 2023-10-09T04:16:44.080964+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:44 smithi153 bash[27868]: audit 2023-10-09T04:16:44.088465+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:44 smithi153 bash[27868]: audit 2023-10-09T04:16:44.090715+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:44.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:44 smithi153 bash[27868]: audit 2023-10-09T04:16:44.092088+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[27462]: audit 2023-10-09T04:16:43.877108+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.82:0/4248736776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[27462]: audit 2023-10-09T04:16:44.080964+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[27462]: audit 2023-10-09T04:16:44.088465+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[27462]: audit 2023-10-09T04:16:44.090715+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[27462]: audit 2023-10-09T04:16:44.092088+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[33901]: audit 2023-10-09T04:16:43.877108+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.82:0/4248736776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[33901]: audit 2023-10-09T04:16:44.080964+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[33901]: audit 2023-10-09T04:16:44.088465+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[33901]: audit 2023-10-09T04:16:44.090715+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:44.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:44 smithi082 bash[33901]: audit 2023-10-09T04:16:44.092088+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: cluster 2023-10-09T04:16:43.286780+0000 mgr.y (mgr.15897) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: audit 2023-10-09T04:16:44.762875+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: audit 2023-10-09T04:16:44.769228+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: audit 2023-10-09T04:16:44.814593+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: audit 2023-10-09T04:16:44.821521+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.451 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:45 smithi153 bash[27868]: audit 2023-10-09T04:16:44.827183+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: cluster 2023-10-09T04:16:43.286780+0000 mgr.y (mgr.15897) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: audit 2023-10-09T04:16:44.762875+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: audit 2023-10-09T04:16:44.769228+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: audit 2023-10-09T04:16:44.814593+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: audit 2023-10-09T04:16:44.821521+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[33901]: audit 2023-10-09T04:16:44.827183+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: cluster 2023-10-09T04:16:43.286780+0000 mgr.y (mgr.15897) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: audit 2023-10-09T04:16:44.762875+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: audit 2023-10-09T04:16:44.769228+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: audit 2023-10-09T04:16:44.814593+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: audit 2023-10-09T04:16:44.821521+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:45.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:45 smithi082 bash[27462]: audit 2023-10-09T04:16:44.827183+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:46.462 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.462 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:16:46.463 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:46 smithi153 bash[27868]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[27462]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:16:46.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:16:46.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:46 smithi082 bash[33901]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:16:47.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:47 smithi082 bash[27462]: cluster 2023-10-09T04:16:45.287796+0000 mgr.y (mgr.15897) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:47.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:47 smithi082 bash[27462]: audit 2023-10-09T04:16:46.325494+0000 mon.a (mon.0) 954 : audit [DBG] from='client.? 172.21.15.82:0/3169031395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:47.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:47 smithi082 bash[33901]: cluster 2023-10-09T04:16:45.287796+0000 mgr.y (mgr.15897) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:47.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:47 smithi082 bash[33901]: audit 2023-10-09T04:16:46.325494+0000 mon.a (mon.0) 954 : audit [DBG] from='client.? 172.21.15.82:0/3169031395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:47.639 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:47 smithi153 bash[27868]: cluster 2023-10-09T04:16:45.287796+0000 mgr.y (mgr.15897) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:47.639 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:47 smithi153 bash[27868]: audit 2023-10-09T04:16:46.325494+0000 mon.a (mon.0) 954 : audit [DBG] from='client.? 172.21.15.82:0/3169031395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:48.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:48 smithi153 bash[27868]: cluster 2023-10-09T04:16:47.288426+0000 mgr.y (mgr.15897) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:48 smithi082 bash[27462]: cluster 2023-10-09T04:16:47.288426+0000 mgr.y (mgr.15897) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:48.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:48 smithi082 bash[33901]: cluster 2023-10-09T04:16:47.288426+0000 mgr.y (mgr.15897) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:49.176 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:48 smithi153 systemd[1]: Stopping Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:48 smithi153 bash[55091]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.052Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T04:16:49.177 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.053Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:16:49.178 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.054Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:16:49.178 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.054Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T04:16:49.178 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[53233]: ts=2023-10-09T04:16:49.054Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T04:16:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:49 smithi082 bash[27462]: audit 2023-10-09T04:16:48.769273+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.82:0/190222807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:49 smithi082 bash[33901]: audit 2023-10-09T04:16:48.769273+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.82:0/190222807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:49.534 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[27868]: audit 2023-10-09T04:16:48.769273+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.82:0/190222807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:49.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[55125]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:16:49.807 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 bash[55187]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T04:16:49.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:16:49.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 systemd[1]: Stopped Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:49.808 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:49 smithi153 systemd[1]: Started Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: cluster 2023-10-09T04:16:49.289479+0000 mgr.y (mgr.15897) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: audit 2023-10-09T04:16:49.782391+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: audit 2023-10-09T04:16:49.791281+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: audit 2023-10-09T04:16:49.837240+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: audit 2023-10-09T04:16:49.838376+0000 mgr.y (mgr.15897) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:50 smithi153 bash[27868]: audit 2023-10-09T04:16:49.846396+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:51.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: cluster 2023-10-09T04:16:49.289479+0000 mgr.y (mgr.15897) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: audit 2023-10-09T04:16:49.782391+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: audit 2023-10-09T04:16:49.791281+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: audit 2023-10-09T04:16:49.837240+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: audit 2023-10-09T04:16:49.838376+0000 mgr.y (mgr.15897) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[27462]: audit 2023-10-09T04:16:49.846396+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: cluster 2023-10-09T04:16:49.289479+0000 mgr.y (mgr.15897) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: audit 2023-10-09T04:16:49.782391+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: audit 2023-10-09T04:16:49.791281+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:51.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: audit 2023-10-09T04:16:49.837240+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: audit 2023-10-09T04:16:49.838376+0000 mgr.y (mgr.15897) 27 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:16:51.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:50 smithi082 bash[33901]: audit 2023-10-09T04:16:49.846396+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.224Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.224Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.224Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi153 (none))" 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.224Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.224Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.228Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.228Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T04:16:51.371 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.230Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T04:16:51.372 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.233Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:16:51.372 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.233Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.962µs 2023-10-09T04:16:51.372 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.233Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:16:51.622 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.368Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T04:16:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[27868]: audit 2023-10-09T04:16:51.211112+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.82:0/2571788707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:51.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.619Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T04:16:51.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.660Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.660Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.660Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.573µs wal_replay_duration=427.752094ms total_replay_duration=427.80755ms 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.667Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.667Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.667Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:16:51.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.693Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.44569ms db_storage=1.207µs remote_storage=2.765µs web_handler=600ns query_engine=1.312µs scrape=514.548µs scrape_sd=99.958µs notify=14.376µs notify_sd=11.507µs rules=25.261342ms 2023-10-09T04:16:51.928 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:16:51 smithi153 bash[55216]: ts=2023-10-09T04:16:51.693Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T04:16:52.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:51 smithi082 bash[27462]: audit 2023-10-09T04:16:51.211112+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.82:0/2571788707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:52.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:51 smithi082 bash[33901]: audit 2023-10-09T04:16:51.211112+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.82:0/2571788707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: cluster 2023-10-09T04:16:51.290213+0000 mgr.y (mgr.15897) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:16:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: audit 2023-10-09T04:16:52.171561+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: audit 2023-10-09T04:16:52.173512+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: audit 2023-10-09T04:16:52.175226+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:52 smithi153 bash[27868]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: cluster 2023-10-09T04:16:51.290213+0000 mgr.y (mgr.15897) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: audit 2023-10-09T04:16:52.171561+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: audit 2023-10-09T04:16:52.173512+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: audit 2023-10-09T04:16:52.175226+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:53.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[27462]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: cluster 2023-10-09T04:16:51.290213+0000 mgr.y (mgr.15897) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: audit 2023-10-09T04:16:52.171561+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: audit 2023-10-09T04:16:52.173512+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: audit 2023-10-09T04:16:52.175226+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:53.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:52 smithi082 bash[33901]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm [INF] Deploying daemon mgr.x on smithi153 2023-10-09T04:16:53.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:53 smithi153 bash[27868]: audit 2023-10-09T04:16:53.653998+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.82:0/391541312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:54.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:53 smithi082 bash[27462]: audit 2023-10-09T04:16:53.653998+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.82:0/391541312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:53 smithi082 bash[33901]: audit 2023-10-09T04:16:53.653998+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.82:0/391541312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:54.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:54 smithi153 systemd[1]: Stopping Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:16:55.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:55 smithi153 bash[27868]: cluster 2023-10-09T04:16:53.290892+0000 mgr.y (mgr.15897) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:55.424 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 bash[55518]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:16:55.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:55 smithi082 bash[33901]: cluster 2023-10-09T04:16:53.290892+0000 mgr.y (mgr.15897) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:55.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:55 smithi082 bash[27462]: cluster 2023-10-09T04:16:53.290892+0000 mgr.y (mgr.15897) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:55.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:16:55.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 bash[55581]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T04:16:55.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:16:55.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 systemd[1]: Stopped Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:56.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:55 smithi153 systemd[1]: Started Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:16:56.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[55637]: debug 2023-10-09T04:16:56.781+0000 7f24a7dc8700 1 -- 172.21.15.153:0/1172793604 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f81fa251e0 con 0x55f8207a4c00 2023-10-09T04:16:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[27868]: cluster 2023-10-09T04:16:55.291981+0000 mgr.y (mgr.15897) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[27868]: audit 2023-10-09T04:16:55.731026+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[27868]: audit 2023-10-09T04:16:55.738167+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[27868]: audit 2023-10-09T04:16:55.749910+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:56.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[27868]: audit 2023-10-09T04:16:56.097429+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.82:0/1641571666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:56.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[27462]: cluster 2023-10-09T04:16:55.291981+0000 mgr.y (mgr.15897) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:56.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[27462]: audit 2023-10-09T04:16:55.731026+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[27462]: audit 2023-10-09T04:16:55.738167+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[27462]: audit 2023-10-09T04:16:55.749910+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:56.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[27462]: audit 2023-10-09T04:16:56.097429+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.82:0/1641571666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:56.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[33901]: cluster 2023-10-09T04:16:55.291981+0000 mgr.y (mgr.15897) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:56.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[33901]: audit 2023-10-09T04:16:55.731026+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[33901]: audit 2023-10-09T04:16:55.738167+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:16:56.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[33901]: audit 2023-10-09T04:16:55.749910+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:56.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:56 smithi082 bash[33901]: audit 2023-10-09T04:16:56.097429+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.82:0/1641571666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:57.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:56 smithi153 bash[55637]: debug 2023-10-09T04:16:56.941+0000 7f24b1f16200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:16:58.019 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:57 smithi153 bash[27868]: audit 2023-10-09T04:16:57.407420+0000 mon.a (mon.0) 965 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:58.019 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:57 smithi153 bash[55637]: debug 2023-10-09T04:16:57.901+0000 7f24b1f16200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:16:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:57 smithi082 bash[27462]: audit 2023-10-09T04:16:57.407420+0000 mon.a (mon.0) 965 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:58.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:57 smithi082 bash[33901]: audit 2023-10-09T04:16:57.407420+0000 mon.a (mon.0) 965 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:58.401 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[55637]: debug 2023-10-09T04:16:58.013+0000 7f24b1f16200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:16:58.402 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[55637]: debug 2023-10-09T04:16:58.121+0000 7f24b1f16200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:16:58.402 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[55637]: debug 2023-10-09T04:16:58.393+0000 7f24b1f16200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:16:59.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[27868]: cluster 2023-10-09T04:16:57.292623+0000 mgr.y (mgr.15897) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:59.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[27868]: audit 2023-10-09T04:16:58.545812+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.82:0/3259228530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:59.145 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[55637]: debug 2023-10-09T04:16:58.757+0000 7f24b1f16200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:16:59.145 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:58 smithi153 bash[55637]: debug 2023-10-09T04:16:58.917+0000 7f24b1f16200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:16:59.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:58 smithi082 bash[27462]: cluster 2023-10-09T04:16:57.292623+0000 mgr.y (mgr.15897) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:59.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:16:58 smithi082 bash[27462]: audit 2023-10-09T04:16:58.545812+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.82:0/3259228530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:59.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:58 smithi082 bash[33901]: cluster 2023-10-09T04:16:57.292623+0000 mgr.y (mgr.15897) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:16:59.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:16:58 smithi082 bash[33901]: audit 2023-10-09T04:16:58.545812+0000 mon.a (mon.0) 966 : audit [DBG] from='client.? 172.21.15.82:0/3259228530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:16:59.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:16:59 smithi153 bash[55637]: debug 2023-10-09T04:16:59.137+0000 7f24b1f16200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:16:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:16:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:16:59] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.33.4" 2023-10-09T04:17:00.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:00 smithi153 bash[55637]: debug 2023-10-09T04:17:00.157+0000 7f24b1f16200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:17:00.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:00 smithi153 bash[55637]: debug 2023-10-09T04:17:00.277+0000 7f24b1f16200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:17:00.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:00 smithi153 bash[55637]: debug 2023-10-09T04:17:00.393+0000 7f24b1f16200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:17:01.161 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:00 smithi153 bash[55637]: debug 2023-10-09T04:17:00.901+0000 7f24b1f16200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:17:01.161 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:01 smithi153 bash[55637]: debug 2023-10-09T04:17:01.153+0000 7f24b1f16200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:17:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:01 smithi153 bash[27868]: cluster 2023-10-09T04:16:59.293749+0000 mgr.y (mgr.15897) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:01.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:01 smithi153 bash[27868]: audit 2023-10-09T04:17:00.989330+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.82:0/2841666903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:01 smithi082 bash[27462]: cluster 2023-10-09T04:16:59.293749+0000 mgr.y (mgr.15897) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:01 smithi082 bash[27462]: audit 2023-10-09T04:17:00.989330+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.82:0/2841666903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:01 smithi082 bash[33901]: cluster 2023-10-09T04:16:59.293749+0000 mgr.y (mgr.15897) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:01 smithi082 bash[33901]: audit 2023-10-09T04:17:00.989330+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.82:0/2841666903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:01.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:01 smithi153 bash[55637]: debug 2023-10-09T04:17:01.661+0000 7f24b1f16200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:17:02.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[55637]: debug 2023-10-09T04:17:02.229+0000 7f24b1f16200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:17:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[27868]: cluster 2023-10-09T04:17:01.294388+0000 mgr.y (mgr.15897) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[27868]: audit 2023-10-09T04:17:01.899374+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[27868]: audit 2023-10-09T04:17:01.906112+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[27868]: audit 2023-10-09T04:17:02.750103+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[27868]: audit 2023-10-09T04:17:02.757238+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.177 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[55637]: debug 2023-10-09T04:17:02.861+0000 7f24b1f16200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:17:03.177 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:02 smithi153 bash[55637]: debug 2023-10-09T04:17:02.989+0000 7f24b1f16200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[33901]: cluster 2023-10-09T04:17:01.294388+0000 mgr.y (mgr.15897) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[33901]: audit 2023-10-09T04:17:01.899374+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[33901]: audit 2023-10-09T04:17:01.906112+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[33901]: audit 2023-10-09T04:17:02.750103+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[33901]: audit 2023-10-09T04:17:02.757238+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[27462]: cluster 2023-10-09T04:17:01.294388+0000 mgr.y (mgr.15897) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[27462]: audit 2023-10-09T04:17:01.899374+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[27462]: audit 2023-10-09T04:17:01.906112+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[27462]: audit 2023-10-09T04:17:02.750103+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:02 smithi082 bash[27462]: audit 2023-10-09T04:17:02.757238+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:03.630 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[55637]: debug 2023-10-09T04:17:03.365+0000 7f24b1f16200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:17:03.630 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[55637]: debug 2023-10-09T04:17:03.477+0000 7f24b1f16200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:17:03.907 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[55637]: debug 2023-10-09T04:17:03.621+0000 7f24b1f16200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:17:03.908 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[55637]: debug 2023-10-09T04:17:03.741+0000 7f24b1f16200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:17:03.908 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[55637]: debug 2023-10-09T04:17:03.869+0000 7f24b1f16200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:17:04.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:04 smithi153 bash[55637]: debug 2023-10-09T04:17:04.065+0000 7f24b1f16200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:17:04.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:03 smithi153 bash[27868]: audit 2023-10-09T04:17:03.441771+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.82:0/3235661442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:04.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:03 smithi082 bash[27462]: audit 2023-10-09T04:17:03.441771+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.82:0/3235661442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:04.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:03 smithi082 bash[33901]: audit 2023-10-09T04:17:03.441771+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.82:0/3235661442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:04.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:04 smithi153 bash[55637]: debug 2023-10-09T04:17:04.197+0000 7f24b1f16200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:17:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[27868]: cluster 2023-10-09T04:17:03.295047+0000 mgr.y (mgr.15897) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:05.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:05 smithi082 bash[33901]: cluster 2023-10-09T04:17:03.295047+0000 mgr.y (mgr.15897) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:05.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:05 smithi082 bash[27462]: cluster 2023-10-09T04:17:03.295047+0000 mgr.y (mgr.15897) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:05.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: debug 2023-10-09T04:17:05.437+0000 7f24b1f16200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:17:05.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: [09/Oct/2023:04:17:05] ENGINE Bus STARTING 2023-10-09T04:17:05.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: CherryPy Checker: 2023-10-09T04:17:05.927 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: The Application mounted at '' has an empty config. 2023-10-09T04:17:05.927 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: [09/Oct/2023:04:17:05] ENGINE Serving on http://:::9283 2023-10-09T04:17:05.927 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:05 smithi153 bash[55637]: [09/Oct/2023:04:17:05] ENGINE Bus STARTED 2023-10-09T04:17:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: cluster 2023-10-09T04:17:05.462086+0000 mon.a (mon.0) 973 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:17:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: cluster 2023-10-09T04:17:05.462322+0000 mon.a (mon.0) 974 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:17:06.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: audit 2023-10-09T04:17:05.468478+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:17:06.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: audit 2023-10-09T04:17:05.470227+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:17:06.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: audit 2023-10-09T04:17:05.475618+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:17:06.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: audit 2023-10-09T04:17:05.476589+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:17:06.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:06 smithi153 bash[27868]: audit 2023-10-09T04:17:05.886156+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.82:0/645557178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: cluster 2023-10-09T04:17:05.462086+0000 mon.a (mon.0) 973 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: cluster 2023-10-09T04:17:05.462322+0000 mon.a (mon.0) 974 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: audit 2023-10-09T04:17:05.468478+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: audit 2023-10-09T04:17:05.470227+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: audit 2023-10-09T04:17:05.475618+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: audit 2023-10-09T04:17:05.476589+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[27462]: audit 2023-10-09T04:17:05.886156+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.82:0/645557178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: cluster 2023-10-09T04:17:05.462086+0000 mon.a (mon.0) 973 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: cluster 2023-10-09T04:17:05.462322+0000 mon.a (mon.0) 974 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: audit 2023-10-09T04:17:05.468478+0000 mon.a (mon.0) 975 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: audit 2023-10-09T04:17:05.470227+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: audit 2023-10-09T04:17:05.475618+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: audit 2023-10-09T04:17:05.476589+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.153:0/2708273563' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:17:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:06 smithi082 bash[33901]: audit 2023-10-09T04:17:05.886156+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.82:0/645557178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:07 smithi153 bash[27868]: cluster 2023-10-09T04:17:05.296226+0000 mgr.y (mgr.15897) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:07.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:07 smithi153 bash[27868]: cluster 2023-10-09T04:17:06.178197+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T04:17:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:07 smithi082 bash[27462]: cluster 2023-10-09T04:17:05.296226+0000 mgr.y (mgr.15897) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:07 smithi082 bash[27462]: cluster 2023-10-09T04:17:06.178197+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T04:17:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:07 smithi082 bash[33901]: cluster 2023-10-09T04:17:05.296226+0000 mgr.y (mgr.15897) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:07 smithi082 bash[33901]: cluster 2023-10-09T04:17:06.178197+0000 mon.a (mon.0) 980 : cluster [DBG] mgrmap e30: y(active, since 38s), standbys: x 2023-10-09T04:17:08.831 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:08 smithi153 bash[55216]: ts=2023-10-09T04:17:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:09.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:09 smithi082 bash[27462]: cluster 2023-10-09T04:17:07.296892+0000 mgr.y (mgr.15897) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:09.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:09 smithi082 bash[27462]: audit 2023-10-09T04:17:08.336696+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.82:0/1078102869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:09.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:09 smithi082 bash[33901]: cluster 2023-10-09T04:17:07.296892+0000 mgr.y (mgr.15897) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:09.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:09 smithi082 bash[33901]: audit 2023-10-09T04:17:08.336696+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.82:0/1078102869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:09 smithi153 bash[27868]: cluster 2023-10-09T04:17:07.296892+0000 mgr.y (mgr.15897) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:09 smithi153 bash[27868]: audit 2023-10-09T04:17:08.336696+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.82:0/1078102869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:09.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:09] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.33.4" 2023-10-09T04:17:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:11 smithi153 bash[27868]: cluster 2023-10-09T04:17:09.297999+0000 mgr.y (mgr.15897) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:11 smithi153 bash[27868]: audit 2023-10-09T04:17:10.778063+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.82:0/380906245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:11.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:11 smithi082 bash[27462]: cluster 2023-10-09T04:17:09.297999+0000 mgr.y (mgr.15897) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:11.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:11 smithi082 bash[27462]: audit 2023-10-09T04:17:10.778063+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.82:0/380906245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:11.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:11 smithi082 bash[33901]: cluster 2023-10-09T04:17:09.297999+0000 mgr.y (mgr.15897) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:11.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:11 smithi082 bash[33901]: audit 2023-10-09T04:17:10.778063+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.82:0/380906245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: cluster 2023-10-09T04:17:11.298687+0000 mgr.y (mgr.15897) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.855867+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.864889+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.866245+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.867619+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.877327+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.960580+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.965639+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:13.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.977421+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.978989+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.986571+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.987889+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.996909+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:11.999621+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:12.001096+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:17:13.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:12 smithi153 bash[27868]: audit 2023-10-09T04:17:12.407159+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:13.201 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: cluster 2023-10-09T04:17:11.298687+0000 mgr.y (mgr.15897) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.855867+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.864889+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.866245+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.867619+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.877327+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.960580+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.965639+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:17:13.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.977421+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.978989+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.986571+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.987889+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.996909+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:11.999621+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:12.001096+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[27462]: audit 2023-10-09T04:17:12.407159+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: cluster 2023-10-09T04:17:11.298687+0000 mgr.y (mgr.15897) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:13.203 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.855867+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.864889+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.866245+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.867619+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.877327+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.960580+0000 mon.a (mon.0) 987 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.965639+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.977421+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.978989+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.986571+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.987889+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:17:13.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.996909+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:17:13.205 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:11.999621+0000 mon.a (mon.0) 994 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:13.205 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:12.001096+0000 mon.a (mon.0) 995 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:17:13.205 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T04:17:13.205 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:12 smithi082 bash[33901]: audit 2023-10-09T04:17:12.407159+0000 mon.a (mon.0) 996 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:13.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:13 smithi082 bash[33901]: audit 2023-10-09T04:17:13.232238+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.82:0/3930882457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:13.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:13 smithi082 bash[27462]: audit 2023-10-09T04:17:13.232238+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.82:0/3930882457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:13 smithi153 bash[27868]: audit 2023-10-09T04:17:13.232238+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.82:0/3930882457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:15 smithi153 bash[27868]: cluster 2023-10-09T04:17:13.299204+0000 mgr.y (mgr.15897) 43 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:15 smithi153 bash[27868]: audit 2023-10-09T04:17:14.333935+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:15.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:15 smithi153 bash[27868]: audit 2023-10-09T04:17:14.335707+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:15.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:15 smithi153 bash[27868]: audit 2023-10-09T04:17:14.336950+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:15.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:15 smithi153 bash[27868]: audit 2023-10-09T04:17:14.338085+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:15.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[27462]: cluster 2023-10-09T04:17:13.299204+0000 mgr.y (mgr.15897) 43 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:15.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[27462]: audit 2023-10-09T04:17:14.333935+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:15.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[27462]: audit 2023-10-09T04:17:14.335707+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:15.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[27462]: audit 2023-10-09T04:17:14.336950+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[27462]: audit 2023-10-09T04:17:14.338085+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[33901]: cluster 2023-10-09T04:17:13.299204+0000 mgr.y (mgr.15897) 43 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[33901]: audit 2023-10-09T04:17:14.333935+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[33901]: audit 2023-10-09T04:17:14.335707+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[33901]: audit 2023-10-09T04:17:14.336950+0000 mon.a (mon.0) 999 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:15.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:15 smithi082 bash[33901]: audit 2023-10-09T04:17:14.338085+0000 mon.a (mon.0) 1000 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:16 smithi153 bash[27868]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:17:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:16 smithi153 bash[27868]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm [INF] Deploying daemon mon.a on smithi082 2023-10-09T04:17:16.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:16 smithi153 bash[27868]: audit 2023-10-09T04:17:15.677291+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.82:0/3045630687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[27462]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:17:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[27462]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm [INF] Deploying daemon mon.a on smithi082 2023-10-09T04:17:16.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[27462]: audit 2023-10-09T04:17:15.677291+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.82:0/3045630687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[33901]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T04:17:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[33901]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm [INF] Deploying daemon mon.a on smithi082 2023-10-09T04:17:16.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[33901]: audit 2023-10-09T04:17:15.677291+0000 mon.a (mon.0) 1001 : audit [DBG] from='client.? 172.21.15.82:0/3045630687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 systemd[1]: Stopping Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[27462]: debug 2023-10-09T04:17:16.767+0000 7fdd32d0e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[27462]: debug 2023-10-09T04:17:16.767+0000 7fdd32d0e700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE Bus STOPPING 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:17:16.906 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE Bus STOPPED 2023-10-09T04:17:16.907 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE Bus STARTING 2023-10-09T04:17:17.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE Serving on http://:::9283 2023-10-09T04:17:17.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:16 smithi082 bash[73210]: [09/Oct/2023:04:17:16] ENGINE Bus STARTED 2023-10-09T04:17:17.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:17 smithi082 bash[76913]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:17:17.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:17 smithi082 bash[76987]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T04:17:17.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:17 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:17:17.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:17 smithi082 systemd[1]: Stopped Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:17:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:17 smithi082 systemd[1]: Started Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:17:18.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:18 smithi153 bash[55216]: ts=2023-10-09T04:17:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.023+0000 7f3fd80b0c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:17:19.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.023+0000 7f3fd80b0c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:17:19.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.027+0000 7f3fd80b0c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:17:19.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 0 load: jerasure load: lrc 2023-10-09T04:17:19.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:17:19.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Git sha 0 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: DB SUMMARY 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: DB Session ID: 8RE6NBQF30298NML4VLN 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:17:19.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 958 Bytes 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 14315875 ; 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:17:19.236 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:17:19.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:17:19.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:17:19.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.env: 0x55f030bb2720 2023-10-09T04:17:19.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:17:19.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.info_log: 0x55f03281f480 2023-10-09T04:17:19.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:17:19.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:17:19.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:17:19.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:17:19.240 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.wal_dir: 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:17:19.241 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.write_buffer_manager: 0x55f031aacc80 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:17:19.242 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.row_cache: None 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:17:19.243 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:17:19.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:17:19.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:17:19.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:17:19.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.merge_operator: 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f03281f580) 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: cache_index_and_filter_blocks: 1 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: pin_top_level_index_and_filter: 1 2023-10-09T04:17:19.248 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: index_type: 0 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: data_block_index_type: 0 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: index_shortening: 1 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: checksum: 4 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: no_block_cache: 0 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_cache: 0x55f031b2f1f0 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_cache_name: BinnedLRUCache 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_cache_options: 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: capacity : 536870912 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: num_shard_bits : 4 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: strict_capacity_limit : 0 2023-10-09T04:17:19.249 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: high_pri_pool_ratio: 0.000 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_cache_compressed: (nil) 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: persistent_cache: (nil) 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_size: 4096 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_size_deviation: 10 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_restart_interval: 16 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: index_block_restart_interval: 1 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: metadata_block_size: 4096 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: partition_filters: 0 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: use_delta_encoding: 1 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: filter_policy: bloomfilter 2023-10-09T04:17:19.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: whole_key_filtering: 1 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: verify_compression: 0 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: read_amp_bytes_per_bit: 0 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: format_version: 5 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: enable_index_compression: 1 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: block_align: 0 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: max_auto_readahead_size: 262144 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: prepopulate_block_cache: 0 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: initial_auto_readahead_size: 8192 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:17:19.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:17:19.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:17:19.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:17:19.254 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:17:19.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:17:19.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:17:19.257 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:17:19.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fd80b0c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.031+0000 7f3fcd0cd700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11381, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d02a794c-34df-4703-8b8d-6a336a07c8e8 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825039038296, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.035+0000 7f3fd80b0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.079+0000 7f3fd80b0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825039082751, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 13822376, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11382, "largest_seqno": 14775, "table_properties": {"data_size": 13802822, "index_size": 15397, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3141, "raw_key_size": 24056, "raw_average_key_size": 19, "raw_value_size": 13769922, "raw_average_value_size": 11042, "num_data_blocks": 671, "num_entries": 1247, "num_filter_entries": 1247, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696825039, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d02a794c-34df-4703-8b8d-6a336a07c8e8", "db_session_id": "8RE6NBQF30298NML4VLN", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.079+0000 7f3fd80b0c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825039082873, "job": 1, "event": "recovery_finished"} 2023-10-09T04:17:19.259 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.079+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.079+0000 7f3fd80b0c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fd80b0c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fd80b0c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f031bd0000 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fd80b0c80 4 rocksdb: DB pointer 0x55f031bbe000 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fc68c0700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fc68c0700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: ** DB Stats ** 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:17:19.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: ** Compaction Stats [default] ** 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: L0 1/0 13.18 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.7 0.02 0.00 1 0.021 0 0 0.0 0.0 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: L6 1/0 22.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Sum 2/0 35.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.7 0.02 0.00 1 0.021 0 0 0.0 0.0 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 624.7 0.02 0.00 1 0.021 0 0 0.0 0.0 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: ** Compaction Stats [default] ** 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:17:19.261 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 624.7 0.02 0.00 1 0.021 0 0 0.0 0.0 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Flush(GB): cumulative 0.013, interval 0.013 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Cumulative compaction: 0.01 GB write, 255.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Interval compaction: 0.01 GB write, 255.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:17:19.262 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Block cache BinnedLRUCache@0x55f031b2f1f0#7 capacity: 512.00 MB usage: 19.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.3e-05 secs_since: 0 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Block cache entry stats(count,size,portion): FilterBlock(1,3.28 KB,0.000625849%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.083+0000 7f3fd80b0c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] at bind addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 1 mon.a@-1(???) e3 preinit fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).mds e6 new map 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).mds e6 print_map 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: e6 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: legacy client fscid: 1 2023-10-09T04:17:19.263 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Filesystem 'foo' (1) 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: fs_name foo 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: epoch 5 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: created 2023-10-09T04:11:26.223721+0000 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: modified 2023-10-09T04:11:33.267531+0000 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: tableserver 0 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: root 0 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: session_timeout 60 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: session_autoclose 300 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: max_file_size 1099511627776 2023-10-09T04:17:19.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: required_client_features {} 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: last_failure 0 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: last_failure_osd_epoch 0 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: max_mds 1 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: in 0 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: up {0=14640} 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: failed 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: damaged 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: stopped 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: data_pools [4] 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: metadata_pool 3 2023-10-09T04:17:19.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: inline_data disabled 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: balancer 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: bal_rank_mask -1 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: standby_count_wanted 1 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: [mds.foo.smithi082.juwitc{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: Standby daemons: 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: [mds.foo.smithi153.braqax{-1:24481} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:17:19.266 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:19.267 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:19.267 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.087+0000 7f3fd80b0c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:19.267 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[77063]: debug 2023-10-09T04:17:19.091+0000 7f3fd80b0c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T04:17:19.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:19] "GET /metrics HTTP/1.1" 200 33777 "" "Prometheus/2.33.4" 2023-10-09T04:17:20.669 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T04:17:20.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[55637]: ignoring --setuser ceph since I am not root 2023-10-09T04:17:20.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[55637]: ignoring --setgroup ceph since I am not root 2023-10-09T04:17:20.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[55637]: debug 2023-10-09T04:17:20.633+0000 7f02e254d700 1 -- 172.21.15.153:0/1856811575 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c44df311e0 con 0x55c44ecb0400 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[55637]: debug 2023-10-09T04:17:20.793+0000 7f02ec27d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: audit 2023-10-09T04:17:16.780396+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: audit 2023-10-09T04:17:16.782128+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: audit 2023-10-09T04:17:16.782414+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:17.300451+0000 mgr.y (mgr.15897) 47 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: audit 2023-10-09T04:17:18.190956+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.82:0/254658429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:19.301681+0000 mgr.y (mgr.15897) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.510496+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T04:17:20.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.533202+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.544962+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.545650+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.545688+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.546692+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 53s), standbys: x 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.573537+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 53s), standbys: x 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: cluster 2023-10-09T04:17:20.573568+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T04:17:20.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:20 smithi153 bash[27868]: audit 2023-10-09T04:17:20.573658+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15897 ' entity='' 2023-10-09T04:17:20.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[73210]: ignoring --setuser ceph since I am not root 2023-10-09T04:17:20.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[73210]: ignoring --setgroup ceph since I am not root 2023-10-09T04:17:20.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[73210]: debug 2023-10-09T04:17:20.643+0000 7fed500be700 1 -- 172.21.15.82:0/222980295 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56455bdcb1e0 con 0x56455cb4ac00 2023-10-09T04:17:20.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[73210]: debug 2023-10-09T04:17:20.799+0000 7fed59dee200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:17:20.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: audit 2023-10-09T04:17:16.780396+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: audit 2023-10-09T04:17:16.782128+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: audit 2023-10-09T04:17:16.782414+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.15897 172.21.15.82:0/2283479358' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:17.300451+0000 mgr.y (mgr.15897) 47 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: audit 2023-10-09T04:17:18.190956+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.82:0/254658429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:19.301681+0000 mgr.y (mgr.15897) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.510496+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.533202+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.544962+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.545650+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:17:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.545688+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:17:20.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.546692+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 53s), standbys: x 2023-10-09T04:17:20.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.573537+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 53s), standbys: x 2023-10-09T04:17:20.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: cluster 2023-10-09T04:17:20.573568+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T04:17:20.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:20 smithi082 bash[33901]: audit 2023-10-09T04:17:20.573658+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15897 ' entity='' 2023-10-09T04:17:21.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:17:21.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:21 smithi082 bash[33901]: audit 2023-10-09T04:17:20.649099+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.82:0/3682506641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:21.872 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:21 smithi153 bash[27868]: audit 2023-10-09T04:17:20.649099+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.82:0/3682506641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:21.873 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:21 smithi153 bash[55637]: debug 2023-10-09T04:17:21.757+0000 7f02ec27d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:17:21.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:21 smithi082 bash[73210]: debug 2023-10-09T04:17:21.775+0000 7fed59dee200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:17:21.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:21 smithi082 bash[73210]: debug 2023-10-09T04:17:21.875+0000 7fed59dee200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:17:22.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:21 smithi153 bash[55637]: debug 2023-10-09T04:17:21.865+0000 7f02ec27d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:17:22.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:21 smithi153 bash[55637]: debug 2023-10-09T04:17:21.981+0000 7f02ec27d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:17:22.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:21 smithi082 bash[73210]: debug 2023-10-09T04:17:21.979+0000 7fed59dee200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:17:22.562 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:22 smithi082 bash[73210]: debug 2023-10-09T04:17:22.247+0000 7fed59dee200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:17:22.673 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:22 smithi153 bash[55637]: debug 2023-10-09T04:17:22.277+0000 7f02ec27d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:17:22.915 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:22 smithi082 bash[73210]: debug 2023-10-09T04:17:22.615+0000 7fed59dee200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:17:22.915 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:22 smithi082 bash[73210]: debug 2023-10-09T04:17:22.783+0000 7fed59dee200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:17:22.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:22 smithi153 bash[55637]: debug 2023-10-09T04:17:22.665+0000 7f02ec27d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:17:22.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:22 smithi153 bash[55637]: debug 2023-10-09T04:17:22.841+0000 7f02ec27d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:17:23.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:23 smithi082 bash[73210]: debug 2023-10-09T04:17:23.007+0000 7fed59dee200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:17:23.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:23 smithi153 bash[55637]: debug 2023-10-09T04:17:23.077+0000 7f02ec27d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:17:24.308 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:24 smithi082 bash[73210]: debug 2023-10-09T04:17:24.031+0000 7fed59dee200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:17:24.308 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:24 smithi082 bash[73210]: debug 2023-10-09T04:17:24.143+0000 7fed59dee200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:17:24.308 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:24 smithi082 bash[73210]: debug 2023-10-09T04:17:24.259+0000 7fed59dee200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:17:24.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:24 smithi153 bash[55637]: debug 2023-10-09T04:17:24.169+0000 7f02ec27d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:17:24.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:24 smithi153 bash[55637]: debug 2023-10-09T04:17:24.289+0000 7f02ec27d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:17:24.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:24 smithi153 bash[55637]: debug 2023-10-09T04:17:24.409+0000 7f02ec27d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:17:25.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:24 smithi153 bash[55637]: debug 2023-10-09T04:17:24.889+0000 7f02ec27d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:17:25.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:25 smithi153 bash[55637]: debug 2023-10-09T04:17:25.117+0000 7f02ec27d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:17:25.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:24 smithi082 bash[73210]: debug 2023-10-09T04:17:24.747+0000 7fed59dee200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:17:25.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:24 smithi082 bash[73210]: debug 2023-10-09T04:17:24.979+0000 7fed59dee200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:17:25.733 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:25 smithi082 bash[73210]: debug 2023-10-09T04:17:25.435+0000 7fed59dee200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:17:25.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:25 smithi153 bash[55637]: debug 2023-10-09T04:17:25.569+0000 7f02ec27d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:17:26.233 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:25 smithi082 bash[73210]: debug 2023-10-09T04:17:25.939+0000 7fed59dee200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:17:26.426 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:26 smithi153 bash[55637]: debug 2023-10-09T04:17:26.073+0000 7f02ec27d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:17:26.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:26 smithi153 bash[55637]: debug 2023-10-09T04:17:26.640+0000 7f02ec27d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:17:26.926 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:26 smithi153 bash[55637]: debug 2023-10-09T04:17:26.752+0000 7f02ec27d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:17:26.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:26 smithi082 bash[73210]: debug 2023-10-09T04:17:26.519+0000 7fed59dee200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:17:26.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:26 smithi082 bash[73210]: debug 2023-10-09T04:17:26.635+0000 7fed59dee200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:17:27.256 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:26 smithi082 bash[73210]: debug 2023-10-09T04:17:26.983+0000 7fed59dee200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:17:27.257 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.099+0000 7fed59dee200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:17:27.257 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.251+0000 7fed59dee200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:17:27.374 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.104+0000 7f02ec27d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:17:27.374 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.220+0000 7f02ec27d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:17:27.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.368+0000 7f02ec27d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:17:27.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.488+0000 7f02ec27d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:17:27.676 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.612+0000 7f02ec27d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:17:27.706 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.367+0000 7fed59dee200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:17:27.706 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.495+0000 7fed59dee200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:17:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.699+0000 7fed59dee200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:17:27.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:27 smithi082 bash[73210]: debug 2023-10-09T04:17:27.831+0000 7fed59dee200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:17:28.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.804+0000 7f02ec27d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:17:28.176 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:27 smithi153 bash[55637]: debug 2023-10-09T04:17:27.936+0000 7f02ec27d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:17:28.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:28 smithi153 bash[55216]: ts=2023-10-09T04:17:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:29.358 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[33901]: debug 2023-10-09T04:17:29.107+0000 7fe7e8540700 -1 mon.c@2(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.079+0000 7fed59dee200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: [09/Oct/2023:04:17:29] ENGINE Bus STARTING 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.303+0000 7fecbf198700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.303+0000 7fecbf198700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.303+0000 7fecbf198700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.303+0000 7fecbf198700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.303+0000 7fecbf198700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.307+0000 7fecb918c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.307+0000 7fecb918c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.307+0000 7fecb918c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.359 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.307+0000 7fecb918c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.360 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: debug 2023-10-09T04:17:29.307+0000 7fecb918c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:17:29.421 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[27868]: debug 2023-10-09T04:17:29.104+0000 7fb65667e700 -1 mon.b@1(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:17:29.421 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: debug 2023-10-09T04:17:29.188+0000 7f02ec27d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:17:29.421 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: [09/Oct/2023:04:17:29] ENGINE Bus STARTING 2023-10-09T04:17:29.421 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: CherryPy Checker: 2023-10-09T04:17:29.421 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: The Application mounted at '' has an empty config. 2023-10-09T04:17:29.422 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: [09/Oct/2023:04:17:29] ENGINE Serving on http://:::9283 2023-10-09T04:17:29.422 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 04:17:29 smithi153 bash[55637]: [09/Oct/2023:04:17:29] ENGINE Bus STARTED 2023-10-09T04:17:29.702 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: CherryPy Checker: 2023-10-09T04:17:29.705 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: The Application mounted at '' has an empty config. 2023-10-09T04:17:29.705 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: [09/Oct/2023:04:17:29] ENGINE Serving on http://:::9283 2023-10-09T04:17:29.705 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: [09/Oct/2023:04:17:29] ENGINE Bus STARTED 2023-10-09T04:17:30.145 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:17:30.145 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (2m) - 7m 13.9M - 0.23.0 ba2b418f427c 4994b766acec 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (84s) - 7m 40.6M - 8.3.5 dad864ee21e9 24b93cfba910 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (5m) - 5m 15.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (5m) - 5m 13.0M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443,9283,8765 running (34s) - 14m 74.9M - 18.2.0-598-gb4fa663f 3060897fdf97 b4d6262d244c 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:8443,9283,8765 running (78s) - 16m 457M - 18.2.0-598-gb4fa663f 3060897fdf97 588466e4b0b0 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (16m) - 16m 61.0M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0f2d400a9efd 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 running (14m) - 14m 57.8M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1f2fe6dbb64d 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 running (14m) - 14m 50.9M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 ae4ed267e219 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (2m) - 8m 8424k - 1.3.1 1dbe0e931976 cda947bde3d9 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (108s) - 8m 8987k - 1.3.1 1dbe0e931976 93816ab22ea5 2023-10-09T04:17:30.146 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (13m) - 13m 44.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9b8a7733f2dc 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (13m) - 13m 46.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 97a8f958f136 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (12m) - 12m 44.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7af0967827d4 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (11m) - 11m 42.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 50ddb6f18074 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (11m) - 11m 46.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (10m) - 10m 43.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (9m) - 9m 45.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (9m) - 9m 48.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:17:30.147 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (40s) - 8m 42.0M - 2.33.4 514e6a882f6e be2d59b8aa70 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.105120+0000 mon.a (mon.0) 10 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.105722+0000 mon.a (mon.0) 11 : cluster [INF] Activating manager daemon y 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.113487+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.113815+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00824792s), standbys: x 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.114933+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.115436+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.115676+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.115898+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.116301+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.117081+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.117343+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.118638+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.119098+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.120021+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.120352+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.120649+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:17:30.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.120949+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.121226+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.121497+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.121918+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.122233+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.122378+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.122585+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.123429+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.123962+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.153384+0000 mon.a (mon.0) 32 : cluster [INF] Manager daemon y is now available 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.198700+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.203045+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:30.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.214611+0000 mon.a (mon.0) 35 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.214741+0000 mon.a (mon.0) 36 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.222126+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.223093+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.226856+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.227409+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.237404+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.243403+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.253169+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.430 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:30 smithi153 bash[27868]: audit 2023-10-09T04:17:29.278983+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:17:30.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:29] "GET /metrics HTTP/1.1" 200 31090 "" "Prometheus/2.33.4" 2023-10-09T04:17:30.483 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[73210]: debug 2023-10-09T04:17:30.123+0000 7fed11da6700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T04:17:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.105120+0000 mon.a (mon.0) 10 : cluster [INF] Active manager daemon y restarted 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.105722+0000 mon.a (mon.0) 11 : cluster [INF] Activating manager daemon y 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.113487+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.113815+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00824792s), standbys: x 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.114933+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.115436+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.115676+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.115898+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.116301+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.117081+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi082.juwitc"}]: dispatch 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.117343+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi153.braqax"}]: dispatch 2023-10-09T04:17:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.118638+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.119098+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.120021+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.120352+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.120649+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.120949+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.121226+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.121497+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.121918+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.122233+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.122378+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.122585+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:17:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.123429+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.123962+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.153384+0000 mon.a (mon.0) 32 : cluster [INF] Manager daemon y is now available 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.198700+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.203045+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.214611+0000 mon.a (mon.0) 35 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.214741+0000 mon.a (mon.0) 36 : cluster [DBG] Standby manager daemon x started 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.222126+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.223093+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.226856+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.227409+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.153:0/3875923645' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:17:30.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.237404+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:30.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.243403+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:17:30.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.253169+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:30.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:30 smithi082 bash[33901]: audit 2023-10-09T04:17:29.278983+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:17:31.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:17:31.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cluster 2023-10-09T04:17:29.464270+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:31.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cluster 2023-10-09T04:17:30.122613+0000 mon.a (mon.0) 40 : cluster [DBG] mgrmap e33: y(active, since 1.01704s), standbys: x 2023-10-09T04:17:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cephadm 2023-10-09T04:17:30.413505+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Bus STARTING 2023-10-09T04:17:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cephadm 2023-10-09T04:17:30.536485+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:17:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cephadm 2023-10-09T04:17:30.638606+0000 mgr.y (mgr.34107) 5 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:17:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:31 smithi153 bash[27868]: cephadm 2023-10-09T04:17:30.638960+0000 mgr.y (mgr.34107) 6 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Bus STARTED 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cluster 2023-10-09T04:17:29.464270+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cluster 2023-10-09T04:17:30.122613+0000 mon.a (mon.0) 40 : cluster [DBG] mgrmap e33: y(active, since 1.01704s), standbys: x 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cephadm 2023-10-09T04:17:30.413505+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Bus STARTING 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cephadm 2023-10-09T04:17:30.536485+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cephadm 2023-10-09T04:17:30.638606+0000 mgr.y (mgr.34107) 5 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:17:31.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:31 smithi082 bash[33901]: cephadm 2023-10-09T04:17:30.638960+0000 mgr.y (mgr.34107) 6 : cephadm [INF] [09/Oct/2023:04:17:30] ENGINE Bus STARTED 2023-10-09T04:17:32.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:32 smithi082 bash[33901]: cluster 2023-10-09T04:17:31.121269+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:32.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:32 smithi153 bash[27868]: cluster 2023-10-09T04:17:31.121269+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:33.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:33 smithi153 bash[27868]: cluster 2023-10-09T04:17:32.131831+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T04:17:33.473 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:33 smithi082 bash[33901]: cluster 2023-10-09T04:17:32.131831+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T04:17:33.925 INFO:teuthology.orchestra.run.smithi082.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:34.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:34 smithi082 bash[33901]: cluster 2023-10-09T04:17:33.121933+0000 mgr.y (mgr.34107) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:34.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:34 smithi082 bash[33901]: audit 2023-10-09T04:17:33.921920+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:34.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:34 smithi153 bash[27868]: cluster 2023-10-09T04:17:33.121933+0000 mgr.y (mgr.34107) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:34.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:34 smithi153 bash[27868]: audit 2023-10-09T04:17:33.921920+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:34.697 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:17:35.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:35 smithi153 bash[27868]: audit 2023-10-09T04:17:33.914392+0000 mgr.y (mgr.34107) 9 : audit [DBG] from='client.26075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:35.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:35 smithi153 bash[27868]: cephadm 2023-10-09T04:17:33.915163+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:35.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:35 smithi082 bash[33901]: audit 2023-10-09T04:17:33.914392+0000 mgr.y (mgr.34107) 9 : audit [DBG] from='client.26075 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:35.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:35 smithi082 bash[33901]: cephadm 2023-10-09T04:17:33.915163+0000 mgr.y (mgr.34107) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:36.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:36 smithi153 bash[27868]: cluster 2023-10-09T04:17:35.122561+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:36.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:36 smithi153 bash[27868]: audit 2023-10-09T04:17:35.947970+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:36.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:36 smithi153 bash[27868]: audit 2023-10-09T04:17:35.956120+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:36 smithi082 bash[33901]: cluster 2023-10-09T04:17:35.122561+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:36 smithi082 bash[33901]: audit 2023-10-09T04:17:35.947970+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:36 smithi082 bash[33901]: audit 2023-10-09T04:17:35.956120+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.235764+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.242045+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.756240+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.762178+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.989003+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:37 smithi153 bash[27868]: audit 2023-10-09T04:17:36.996897+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.235764+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.242045+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.756240+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.762178+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.989003+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:37.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:37 smithi082 bash[33901]: audit 2023-10-09T04:17:36.996897+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:38.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:38 smithi153 bash[27868]: cluster 2023-10-09T04:17:37.123586+0000 mgr.y (mgr.34107) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:38.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:38 smithi082 bash[33901]: cluster 2023-10-09T04:17:37.123586+0000 mgr.y (mgr.34107) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:38.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:38 smithi153 bash[55216]: ts=2023-10-09T04:17:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:39.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:39] "GET /metrics HTTP/1.1" 200 31090 "" "Prometheus/2.33.4" 2023-10-09T04:17:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:40 smithi153 bash[27868]: cluster 2023-10-09T04:17:39.124218+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:40.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:40 smithi082 bash[33901]: cluster 2023-10-09T04:17:39.124218+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:42.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:42 smithi082 bash[33901]: cluster 2023-10-09T04:17:41.125332+0000 mgr.y (mgr.34107) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:42.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:42 smithi153 bash[27868]: cluster 2023-10-09T04:17:41.125332+0000 mgr.y (mgr.34107) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:44 smithi082 bash[33901]: cluster 2023-10-09T04:17:43.126036+0000 mgr.y (mgr.34107) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:44 smithi153 bash[27868]: cluster 2023-10-09T04:17:43.126036+0000 mgr.y (mgr.34107) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:45.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:45 smithi153 bash[27868]: audit 2023-10-09T04:17:44.238199+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:45 smithi082 bash[33901]: audit 2023-10-09T04:17:44.238199+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:46 smithi082 bash[33901]: cluster 2023-10-09T04:17:45.126671+0000 mgr.y (mgr.34107) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:46 smithi082 bash[33901]: audit 2023-10-09T04:17:45.775119+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:46 smithi082 bash[33901]: audit 2023-10-09T04:17:45.783017+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:46 smithi082 bash[33901]: audit 2023-10-09T04:17:45.785309+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:46.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:46 smithi082 bash[33901]: audit 2023-10-09T04:17:45.786746+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:46 smithi153 bash[27868]: cluster 2023-10-09T04:17:45.126671+0000 mgr.y (mgr.34107) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:46 smithi153 bash[27868]: audit 2023-10-09T04:17:45.775119+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:46 smithi153 bash[27868]: audit 2023-10-09T04:17:45.783017+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:46 smithi153 bash[27868]: audit 2023-10-09T04:17:45.785309+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:46 smithi153 bash[27868]: audit 2023-10-09T04:17:45.786746+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:45.788444+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:17:47.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:45.788754+0000 mgr.y (mgr.34107) 18 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:45.953385+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:46.003343+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:46.118500+0000 mgr.y (mgr.34107) 21 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:46.168185+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:46.283521+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: cephadm 2023-10-09T04:17:46.333106+0000 mgr.y (mgr.34107) 24 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.456734+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.464064+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.508568+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.519866+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.526050+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.608937+0000 mon.a (mon.0) 61 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:47.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.611622+0000 mon.a (mon.0) 62 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:47.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.612952+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:47.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:47 smithi082 bash[33901]: audit 2023-10-09T04:17:46.621387+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:45.788444+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:45.788754+0000 mgr.y (mgr.34107) 18 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:45.953385+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:46.003343+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:46.118500+0000 mgr.y (mgr.34107) 21 : cephadm [INF] Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:46.168185+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:17:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:46.283521+0000 mgr.y (mgr.34107) 23 : cephadm [INF] Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: cephadm 2023-10-09T04:17:46.333106+0000 mgr.y (mgr.34107) 24 : cephadm [INF] Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.456734+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.464064+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.508568+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.519866+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.526050+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.608937+0000 mon.a (mon.0) 61 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.611622+0000 mon.a (mon.0) 62 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.612952+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:47.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:47 smithi153 bash[27868]: audit 2023-10-09T04:17:46.621387+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:17:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:48 smithi153 bash[27868]: cluster 2023-10-09T04:17:47.127859+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:48.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:48 smithi153 bash[55216]: ts=2023-10-09T04:17:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:48 smithi082 bash[33901]: cluster 2023-10-09T04:17:47.127859+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:49.985 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:49] "GET /metrics HTTP/1.1" 200 33772 "" "Prometheus/2.33.4" 2023-10-09T04:17:50.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:50 smithi153 bash[27868]: cluster 2023-10-09T04:17:49.128581+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:50.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:50 smithi082 bash[33901]: cluster 2023-10-09T04:17:49.128581+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:52.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:52 smithi082 bash[33901]: cluster 2023-10-09T04:17:51.129659+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:52 smithi153 bash[27868]: cluster 2023-10-09T04:17:51.129659+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:54.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:54 smithi082 bash[33901]: cluster 2023-10-09T04:17:53.130306+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:54 smithi153 bash[27868]: cluster 2023-10-09T04:17:53.130306+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:56.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:56 smithi082 bash[33901]: cluster 2023-10-09T04:17:55.131151+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:56 smithi153 bash[27868]: cluster 2023-10-09T04:17:55.131151+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:58.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:17:58 smithi082 bash[33901]: cluster 2023-10-09T04:17:57.132170+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:58.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:17:58 smithi153 bash[27868]: cluster 2023-10-09T04:17:57.132170+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:17:58.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:17:58 smithi153 bash[55216]: ts=2023-10-09T04:17:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:17:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:17:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:17:59] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:18:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:00 smithi153 bash[27868]: cluster 2023-10-09T04:17:59.132808+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:00 smithi153 bash[27868]: audit 2023-10-09T04:17:59.238153+0000 mon.a (mon.0) 65 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:00 smithi082 bash[33901]: cluster 2023-10-09T04:17:59.132808+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:00 smithi082 bash[33901]: audit 2023-10-09T04:17:59.238153+0000 mon.a (mon.0) 65 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:02.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:02 smithi082 bash[33901]: cluster 2023-10-09T04:18:01.133866+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:02 smithi153 bash[27868]: cluster 2023-10-09T04:18:01.133866+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:04.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:04 smithi082 bash[33901]: cluster 2023-10-09T04:18:03.134527+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:04.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:04 smithi153 bash[27868]: cluster 2023-10-09T04:18:03.134527+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:06 smithi082 bash[33901]: cluster 2023-10-09T04:18:05.135633+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:06 smithi153 bash[27868]: cluster 2023-10-09T04:18:05.135633+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:08.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:08 smithi082 bash[33901]: cluster 2023-10-09T04:18:07.136534+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:08.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:08 smithi153 bash[27868]: cluster 2023-10-09T04:18:07.136534+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:08.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:08 smithi153 bash[55216]: ts=2023-10-09T04:18:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:09.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:09] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:18:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:10 smithi153 bash[27868]: cluster 2023-10-09T04:18:09.137196+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:10.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:10 smithi082 bash[33901]: cluster 2023-10-09T04:18:09.137196+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:12.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:12 smithi082 bash[33901]: cluster 2023-10-09T04:18:11.138195+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:12 smithi153 bash[27868]: cluster 2023-10-09T04:18:11.138195+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:14 smithi082 bash[33901]: cluster 2023-10-09T04:18:13.138844+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:14 smithi153 bash[27868]: cluster 2023-10-09T04:18:13.138844+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:15 smithi082 bash[33901]: audit 2023-10-09T04:18:14.239095+0000 mon.a (mon.0) 66 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:15 smithi153 bash[27868]: audit 2023-10-09T04:18:14.239095+0000 mon.a (mon.0) 66 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:16.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:16 smithi082 bash[33901]: cluster 2023-10-09T04:18:15.139721+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:16 smithi153 bash[27868]: cluster 2023-10-09T04:18:15.139721+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:18.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:18 smithi082 bash[33901]: cluster 2023-10-09T04:18:17.140484+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:18 smithi153 bash[27868]: cluster 2023-10-09T04:18:17.140484+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:18.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:18 smithi153 bash[55216]: ts=2023-10-09T04:18:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:19.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:19] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:18:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:20 smithi153 bash[27868]: cluster 2023-10-09T04:18:19.141057+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:20 smithi082 bash[33901]: cluster 2023-10-09T04:18:19.141057+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:22.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:22 smithi082 bash[33901]: cluster 2023-10-09T04:18:21.142104+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:22 smithi153 bash[27868]: cluster 2023-10-09T04:18:21.142104+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:24 smithi082 bash[33901]: cluster 2023-10-09T04:18:23.142763+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:24 smithi153 bash[27868]: cluster 2023-10-09T04:18:23.142763+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:26.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:26 smithi082 bash[33901]: cluster 2023-10-09T04:18:25.143566+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:26 smithi153 bash[27868]: cluster 2023-10-09T04:18:25.143566+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:28.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:28 smithi082 bash[33901]: cluster 2023-10-09T04:18:27.144141+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:28.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:28 smithi153 bash[27868]: cluster 2023-10-09T04:18:27.144141+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:28.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:28 smithi153 bash[55216]: ts=2023-10-09T04:18:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:29.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:29] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:18:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:30 smithi153 bash[27868]: cluster 2023-10-09T04:18:29.144791+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:30 smithi153 bash[27868]: audit 2023-10-09T04:18:29.242628+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:30 smithi082 bash[33901]: cluster 2023-10-09T04:18:29.144791+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:30 smithi082 bash[33901]: audit 2023-10-09T04:18:29.242628+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:32.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:32 smithi082 bash[33901]: cluster 2023-10-09T04:18:31.145978+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:32 smithi153 bash[27868]: cluster 2023-10-09T04:18:31.145978+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:34.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:34 smithi082 bash[33901]: cluster 2023-10-09T04:18:33.146493+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:34 smithi153 bash[27868]: cluster 2023-10-09T04:18:33.146493+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:36 smithi082 bash[33901]: cluster 2023-10-09T04:18:35.147170+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:36 smithi153 bash[27868]: cluster 2023-10-09T04:18:35.147170+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:38.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:18:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:38 smithi082 bash[33901]: cluster 2023-10-09T04:18:37.147740+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:38.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:38 smithi153 bash[27868]: cluster 2023-10-09T04:18:37.147740+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:38.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:38 smithi153 bash[55216]: ts=2023-10-09T04:18:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:39.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:39] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:18:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:40 smithi153 bash[27868]: cluster 2023-10-09T04:18:39.148366+0000 mgr.y (mgr.34107) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:40 smithi082 bash[33901]: cluster 2023-10-09T04:18:39.148366+0000 mgr.y (mgr.34107) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:41.510 INFO:teuthology.orchestra.run.smithi082.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:18:42.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T04:18:42.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:42 smithi082 bash[33901]: cluster 2023-10-09T04:18:41.149045+0000 mgr.y (mgr.34107) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:42.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:42 smithi082 bash[33901]: audit 2023-10-09T04:18:41.501738+0000 mgr.y (mgr.34107) 53 : audit [DBG] from='client.26081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:18:42.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:42 smithi082 bash[33901]: cephadm 2023-10-09T04:18:41.502341+0000 mgr.y (mgr.34107) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:18:42.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:42 smithi082 bash[33901]: audit 2023-10-09T04:18:41.507343+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:42.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:42 smithi082 bash[33901]: audit 2023-10-09T04:18:41.521256+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:42.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:42 smithi153 bash[27868]: cluster 2023-10-09T04:18:41.149045+0000 mgr.y (mgr.34107) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:42 smithi153 bash[27868]: audit 2023-10-09T04:18:41.501738+0000 mgr.y (mgr.34107) 53 : audit [DBG] from='client.26081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:18:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:42 smithi153 bash[27868]: cephadm 2023-10-09T04:18:41.502341+0000 mgr.y (mgr.34107) 54 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:18:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:42 smithi153 bash[27868]: audit 2023-10-09T04:18:41.507343+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:42 smithi153 bash[27868]: audit 2023-10-09T04:18:41.521256+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:44 smithi082 bash[33901]: cluster 2023-10-09T04:18:43.149669+0000 mgr.y (mgr.34107) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:44 smithi082 bash[33901]: audit 2023-10-09T04:18:44.243300+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:44 smithi153 bash[27868]: cluster 2023-10-09T04:18:43.149669+0000 mgr.y (mgr.34107) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:44.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:44 smithi153 bash[27868]: audit 2023-10-09T04:18:44.243300+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:44.968 INFO:teuthology.orchestra.run.smithi082.stdout:wait for majority of mons to upgrade 2023-10-09T04:18:45.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T04:18:46.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:46 smithi082 bash[33901]: cluster 2023-10-09T04:18:45.150668+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:46.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:46 smithi153 bash[27868]: cluster 2023-10-09T04:18:45.150668+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: cluster 2023-10-09T04:18:47.151292+0000 mgr.y (mgr.34107) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:47.410351+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:47.417479+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:47.815497+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:47.823839+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.665 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:48.310118+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:48.666 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:48.311477+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:18:48.666 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:48.320561+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.666 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[27868]: audit 2023-10-09T04:18:48.404752+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: cluster 2023-10-09T04:18:47.151292+0000 mgr.y (mgr.34107) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:47.410351+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:47.417479+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:47.815497+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:47.823839+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:48.310118+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:48.311477+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:48.320561+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:48.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:48 smithi082 bash[33901]: audit 2023-10-09T04:18:48.404752+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:48.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:48 smithi153 bash[55216]: ts=2023-10-09T04:18:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:49 smithi153 bash[27868]: audit 2023-10-09T04:18:48.412045+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:18:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:49 smithi153 bash[27868]: audit 2023-10-09T04:18:48.414340+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:18:49.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:49 smithi153 bash[27868]: audit 2023-10-09T04:18:48.415435+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:18:49.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:49 smithi153 bash[27868]: cephadm 2023-10-09T04:18:48.416181+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T04:18:49.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:49 smithi153 bash[27868]: audit 2023-10-09T04:18:49.064580+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.82:0/1551051967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:18:49.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[33901]: audit 2023-10-09T04:18:48.412045+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:18:49.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[33901]: audit 2023-10-09T04:18:48.414340+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:18:49.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[33901]: audit 2023-10-09T04:18:48.415435+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:18:49.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[33901]: cephadm 2023-10-09T04:18:48.416181+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T04:18:49.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[33901]: audit 2023-10-09T04:18:49.064580+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.82:0/1551051967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:18:49.936 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:49] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:18:50.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:50 smithi153 bash[27868]: cluster 2023-10-09T04:18:49.151742+0000 mgr.y (mgr.34107) 59 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:50.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:50 smithi082 bash[33901]: cluster 2023-10-09T04:18:49.151742+0000 mgr.y (mgr.34107) 59 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: cephadm 2023-10-09T04:18:50.635539+0000 mgr.y (mgr.34107) 60 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: audit 2023-10-09T04:18:50.644120+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: audit 2023-10-09T04:18:50.645798+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: audit 2023-10-09T04:18:50.646882+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: audit 2023-10-09T04:18:50.647908+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: cephadm 2023-10-09T04:18:50.649235+0000 mgr.y (mgr.34107) 61 : cephadm [INF] Deploying daemon mon.c on smithi082 2023-10-09T04:18:51.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:51 smithi153 bash[27868]: audit 2023-10-09T04:18:51.505396+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.82:0/4073491519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: cephadm 2023-10-09T04:18:50.635539+0000 mgr.y (mgr.34107) 60 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: audit 2023-10-09T04:18:50.644120+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: audit 2023-10-09T04:18:50.645798+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: audit 2023-10-09T04:18:50.646882+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: audit 2023-10-09T04:18:50.647908+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: cephadm 2023-10-09T04:18:50.649235+0000 mgr.y (mgr.34107) 61 : cephadm [INF] Deploying daemon mon.c on smithi082 2023-10-09T04:18:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:51 smithi082 bash[33901]: audit 2023-10-09T04:18:51.505396+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.82:0/4073491519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:18:52.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:18:52 smithi153 bash[27868]: cluster 2023-10-09T04:18:51.152713+0000 mgr.y (mgr.34107) 62 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:52 smithi082 bash[33901]: cluster 2023-10-09T04:18:51.152713+0000 mgr.y (mgr.34107) 62 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:53.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:53 smithi082 systemd[1]: Stopping Ceph mon.c for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:18:53.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:53 smithi082 bash[33901]: debug 2023-10-09T04:18:53.209+0000 7fe7eed4d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:18:53.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:53 smithi082 bash[33901]: debug 2023-10-09T04:18:53.209+0000 7fe7eed4d700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:18:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:53 smithi082 bash[79586]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T04:18:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:54 smithi082 bash[79676]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T04:18:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:54 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T04:18:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:54 smithi082 systemd[1]: Stopped Ceph mon.c for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:18:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:54 smithi082 systemd[1]: Started Ceph mon.c for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:18:55.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.353+0000 7f86ec655c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.353+0000 7f86ec655c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.353+0000 7f86ec655c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 0 load: jerasure load: lrc 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Git sha 0 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: DB SUMMARY 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: DB Session ID: 7Q4SVLA0P954JBY27LD4 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:18:55.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1072 Bytes 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000021.log size: 1713699 ; 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.env: 0x55ba8b08b720 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:18:55.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.info_log: 0x55ba8ce75480 2023-10-09T04:18:55.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:18:55.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:18:55.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:18:55.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:18:55.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:18:55.487 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.wal_dir: 2023-10-09T04:18:55.488 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.write_buffer_manager: 0x55ba8c102c80 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:18:55.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.row_cache: None 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:18:55.490 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:18:55.491 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:18:55.492 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:18:55.493 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.merge_operator: 2023-10-09T04:18:55.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ba8ce75580) 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: cache_index_and_filter_blocks: 1 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: pin_top_level_index_and_filter: 1 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: index_type: 0 2023-10-09T04:18:55.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: data_block_index_type: 0 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: index_shortening: 1 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: checksum: 4 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: no_block_cache: 0 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_cache: 0x55ba8c1851f0 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_cache_name: BinnedLRUCache 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_cache_options: 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: capacity : 536870912 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: num_shard_bits : 4 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: strict_capacity_limit : 0 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: high_pri_pool_ratio: 0.000 2023-10-09T04:18:55.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_cache_compressed: (nil) 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: persistent_cache: (nil) 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_size: 4096 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_size_deviation: 10 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_restart_interval: 16 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: index_block_restart_interval: 1 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: metadata_block_size: 4096 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: partition_filters: 0 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: use_delta_encoding: 1 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: filter_policy: bloomfilter 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: whole_key_filtering: 1 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: verify_compression: 0 2023-10-09T04:18:55.497 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: read_amp_bytes_per_bit: 0 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: format_version: 5 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: enable_index_compression: 1 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: block_align: 0 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: max_auto_readahead_size: 262144 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: prepopulate_block_cache: 0 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: initial_auto_readahead_size: 8192 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:18:55.498 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:18:55.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:18:55.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:18:55.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:18:55.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:18:55.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:18:55.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:18:55.505 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:18:55.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86ec655c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.357+0000 7f86e1672700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.361+0000 7f86ec655c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 16975, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.361+0000 7f86ec655c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.361+0000 7f86ec655c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 868dce5a-0363-4d5d-a75c-e249bd28d8eb 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.361+0000 7f86ec655c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825135365525, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.361+0000 7f86ec655c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.365+0000 7f86ec655c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825135371149, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 1139074, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 16980, "largest_seqno": 17320, "table_properties": {"data_size": 1136240, "index_size": 1502, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2432, "raw_average_key_size": 21, "raw_value_size": 1132908, "raw_average_value_size": 10115, "num_data_blocks": 65, "num_entries": 112, "num_filter_entries": 112, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696825135, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "868dce5a-0363-4d5d-a75c-e249bd28d8eb", "db_session_id": "7Q4SVLA0P954JBY27LD4", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.365+0000 7f86ec655c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825135371256, "job": 1, "event": "recovery_finished"} 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.365+0000 7f86ec655c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T04:18:55.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ba8c226000 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 4 rocksdb: DB pointer 0x55ba8c214000 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86dae65700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86dae65700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: ** DB Stats ** 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:18:55.508 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: ** Compaction Stats [default] ** 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: L0 1/0 1.09 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: L6 1/0 31.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Sum 2/0 32.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 383.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: ** Compaction Stats [default] ** 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 383.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:18:55.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Cumulative compaction: 0.00 GB write, 99.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Interval compaction: 0.00 GB write, 99.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Block cache BinnedLRUCache@0x55ba8c1851f0#7 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2023-10-09T04:18:55.510 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] at bind addrs [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 1 mon.c@-1(???) e3 preinit fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 0 mon.c@-1(???).mds e6 new map 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.369+0000 7f86ec655c80 0 mon.c@-1(???).mds e6 print_map 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: e6 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: legacy client fscid: 1 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Filesystem 'foo' (1) 2023-10-09T04:18:55.511 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: fs_name foo 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: epoch 5 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: created 2023-10-09T04:11:26.223721+0000 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: modified 2023-10-09T04:11:33.267531+0000 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: tableserver 0 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: root 0 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: session_timeout 60 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: session_autoclose 300 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: max_file_size 1099511627776 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: required_client_features {} 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: last_failure 0 2023-10-09T04:18:55.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: last_failure_osd_epoch 0 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: max_mds 1 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: in 0 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: up {0=14640} 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: failed 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: damaged 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: stopped 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: data_pools [4] 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: metadata_pool 3 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: inline_data disabled 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: balancer 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: bal_rank_mask -1 2023-10-09T04:18:55.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: standby_count_wanted 1 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: [mds.foo.smithi082.juwitc{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: Standby daemons: 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: [mds.foo.smithi153.braqax{-1:24481} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 0 mon.c@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T04:18:55.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T04:18:55.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:18:55 smithi082 bash[79733]: debug 2023-10-09T04:18:55.373+0000 7f86ec655c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T04:18:58.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:18:58 smithi153 bash[55216]: ts=2023-10-09T04:18:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:18:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:18:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:18:59] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.156642+0000 mgr.y (mgr.34107) 67 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.617503+0000 mon.b (mon.1) 110 : cluster [INF] mon.b calling monitor election 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.617747+0000 mon.a (mon.0) 101 : cluster [INF] mon.a calling monitor election 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.642609+0000 mon.a (mon.0) 102 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.650602+0000 mon.a (mon.0) 103 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:19:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.651508+0000 mon.a (mon.0) 104 : cluster [DBG] fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:19:02.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.651560+0000 mon.a (mon.0) 105 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:19:02.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.652057+0000 mon.a (mon.0) 106 : cluster [DBG] mgrmap e34: y(active, since 92s), standbys: x 2023-10-09T04:19:02.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.652310+0000 mon.a (mon.0) 107 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:19:02.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:01 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.652346+0000 mon.a (mon.0) 108 : cluster [INF] Cluster is now healthy 2023-10-09T04:19:02.433 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:19:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:02 smithi153 bash[27868]: cluster 2023-10-09T04:18:55.590970+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T04:19:03.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:02 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.678877+0000 mon.a (mon.0) 109 : cluster [INF] overall HEALTH_OK 2023-10-09T04:19:03.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:02 smithi153 bash[27868]: cluster 2023-10-09T04:19:01.698062+0000 mon.a (mon.0) 110 : cluster [DBG] mgrmap e35: y(active, since 92s), standbys: x 2023-10-09T04:19:03.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:02 smithi153 bash[27868]: audit 2023-10-09T04:19:02.410861+0000 mon.a (mon.0) 111 : audit [DBG] from='client.? 172.21.15.82:0/1888849131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:19:03.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:19:04.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:03 smithi153 bash[27868]: cluster 2023-10-09T04:19:03.157296+0000 mgr.y (mgr.34107) 68 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:05.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:05 smithi082 bash[73210]: debug 2023-10-09T04:19:05.381+0000 7fed11da6700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (3m) - 9m 14.0M - 0.23.0 ba2b418f427c 4994b766acec 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (3m) - 8m 41.1M - 8.3.5 dad864ee21e9 24b93cfba910 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (7m) - 7m 15.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (7m) - 7m 13.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443,9283,8765 running (2m) - 16m 426M - 18.2.0-598-gb4fa663f 3060897fdf97 b4d6262d244c 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:8443,9283,8765 running (2m) - 18m 473M - 18.2.0-598-gb4fa663f 3060897fdf97 588466e4b0b0 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (108s) - 18m 46.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 93a348625d31 2023-10-09T04:19:06.442 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 running (16m) - 16m 53.1M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 1f2fe6dbb64d 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 starting - - - 2048M 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (3m) - 9m 8500k - 1.3.1 1dbe0e931976 cda947bde3d9 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (3m) - 9m 9415k - 1.3.1 1dbe0e931976 93816ab22ea5 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (15m) - 15m 45.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9b8a7733f2dc 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (14m) - 14m 47.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 97a8f958f136 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (14m) - 14m 44.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7af0967827d4 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (13m) - 13m 42.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 50ddb6f18074 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (12m) - 12m 46.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (12m) - 12m 44.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (11m) - 11m 45.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (10m) - 10m 49.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:19:06.443 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (2m) - 9m 54.7M - 2.33.4 514e6a882f6e be2d59b8aa70 2023-10-09T04:19:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:06 smithi153 bash[27868]: cluster 2023-10-09T04:19:05.158202+0000 mgr.y (mgr.34107) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:07.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:19:07.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:07 smithi153 bash[27868]: audit 2023-10-09T04:19:06.424727+0000 mgr.y (mgr.34107) 70 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:19:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:07 smithi153 bash[27868]: audit 2023-10-09T04:19:06.830735+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:07 smithi153 bash[27868]: audit 2023-10-09T04:19:06.838044+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:08.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:08 smithi153 bash[27868]: cluster 2023-10-09T04:19:07.158824+0000 mgr.y (mgr.34107) 71 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:08 smithi153 bash[27868]: audit 2023-10-09T04:19:07.625103+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:08.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:08 smithi153 bash[27868]: audit 2023-10-09T04:19:07.630811+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:08.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:08 smithi153 bash[55216]: ts=2023-10-09T04:19:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:09] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.33.4" 2023-10-09T04:19:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:10 smithi153 bash[27868]: cluster 2023-10-09T04:19:09.159426+0000 mgr.y (mgr.34107) 72 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:10.511 INFO:teuthology.orchestra.run.smithi082.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:11.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:19:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:11 smithi153 bash[27868]: audit 2023-10-09T04:19:10.500707+0000 mgr.y (mgr.34107) 73 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:19:11.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:11 smithi153 bash[27868]: cephadm 2023-10-09T04:19:10.501366+0000 mgr.y (mgr.34107) 74 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:11.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:11 smithi153 bash[27868]: audit 2023-10-09T04:19:10.507897+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:12 smithi153 bash[27868]: cluster 2023-10-09T04:19:11.160500+0000 mgr.y (mgr.34107) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:19:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:14 smithi153 bash[27868]: cluster 2023-10-09T04:19:13.161146+0000 mgr.y (mgr.34107) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:14.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:14 smithi153 bash[27868]: audit 2023-10-09T04:19:14.243976+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:16.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:16 smithi153 bash[27868]: cluster 2023-10-09T04:19:15.162126+0000 mgr.y (mgr.34107) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:18.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[55216]: ts=2023-10-09T04:19:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: cluster 2023-10-09T04:19:17.162661+0000 mgr.y (mgr.34107) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:17.932441+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:17.943208+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:17.944936+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:17.946168+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:17.952501+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:18.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:18.035277+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:18.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:18.038188+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:18.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:18.039525+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:18.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:18 smithi153 bash[27868]: audit 2023-10-09T04:19:18.047060+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:19:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:19] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.33.4" 2023-10-09T04:19:20.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:20 smithi153 bash[27868]: cluster 2023-10-09T04:19:19.163299+0000 mgr.y (mgr.34107) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:22 smithi153 bash[27868]: cluster 2023-10-09T04:19:21.164353+0000 mgr.y (mgr.34107) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:24 smithi153 bash[27868]: cluster 2023-10-09T04:19:23.164981+0000 mgr.y (mgr.34107) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:26.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:26 smithi153 bash[27868]: cluster 2023-10-09T04:19:25.165855+0000 mgr.y (mgr.34107) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:28.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:28 smithi153 bash[27868]: cluster 2023-10-09T04:19:27.166527+0000 mgr.y (mgr.34107) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:28.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:28 smithi153 bash[55216]: ts=2023-10-09T04:19:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:29.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:29] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.33.4" 2023-10-09T04:19:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:30 smithi153 bash[27868]: cluster 2023-10-09T04:19:29.167254+0000 mgr.y (mgr.34107) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:30.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:30 smithi153 bash[27868]: audit 2023-10-09T04:19:29.244733+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:32 smithi153 bash[27868]: cluster 2023-10-09T04:19:31.168344+0000 mgr.y (mgr.34107) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:34 smithi153 bash[27868]: cluster 2023-10-09T04:19:33.168965+0000 mgr.y (mgr.34107) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:36 smithi153 bash[27868]: cluster 2023-10-09T04:19:35.169915+0000 mgr.y (mgr.34107) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:38.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:38 smithi153 bash[27868]: cluster 2023-10-09T04:19:37.170534+0000 mgr.y (mgr.34107) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:38.926 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:38 smithi153 bash[55216]: ts=2023-10-09T04:19:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:39] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.33.4" 2023-10-09T04:19:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:40 smithi153 bash[27868]: cluster 2023-10-09T04:19:39.171056+0000 mgr.y (mgr.34107) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:42 smithi153 bash[27868]: cluster 2023-10-09T04:19:41.172097+0000 mgr.y (mgr.34107) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:44.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:44 smithi153 bash[27868]: cluster 2023-10-09T04:19:43.172881+0000 mgr.y (mgr.34107) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:45 smithi153 bash[27868]: audit 2023-10-09T04:19:44.244714+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:46.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:46 smithi153 bash[27868]: cluster 2023-10-09T04:19:45.173965+0000 mgr.y (mgr.34107) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:48.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:48 smithi153 bash[27868]: cluster 2023-10-09T04:19:47.174647+0000 mgr.y (mgr.34107) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:48.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:48 smithi153 bash[55216]: ts=2023-10-09T04:19:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:49] "GET /metrics HTTP/1.1" 200 33779 "" "Prometheus/2.33.4" 2023-10-09T04:19:50.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:50 smithi153 bash[27868]: cluster 2023-10-09T04:19:49.175405+0000 mgr.y (mgr.34107) 94 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:52.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:52 smithi153 bash[27868]: cluster 2023-10-09T04:19:51.176573+0000 mgr.y (mgr.34107) 95 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:54.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:54 smithi153 bash[27868]: cluster 2023-10-09T04:19:53.177175+0000 mgr.y (mgr.34107) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:56.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:56 smithi153 bash[27868]: cluster 2023-10-09T04:19:55.178324+0000 mgr.y (mgr.34107) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:58.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:19:58 smithi153 bash[27868]: cluster 2023-10-09T04:19:57.179042+0000 mgr.y (mgr.34107) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:19:58.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:19:58 smithi153 bash[55216]: ts=2023-10-09T04:19:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:19:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:19:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:19:59] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-09T04:20:00.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:00 smithi153 bash[27868]: cluster 2023-10-09T04:19:59.179774+0000 mgr.y (mgr.34107) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:00.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:00 smithi153 bash[27868]: audit 2023-10-09T04:19:59.245329+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:00.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:00 smithi153 bash[27868]: cluster 2023-10-09T04:20:00.000148+0000 mon.a (mon.0) 130 : cluster [INF] overall HEALTH_OK 2023-10-09T04:20:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:02 smithi153 bash[27868]: cluster 2023-10-09T04:20:01.180938+0000 mgr.y (mgr.34107) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:04 smithi153 bash[27868]: cluster 2023-10-09T04:20:03.181618+0000 mgr.y (mgr.34107) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:06 smithi153 bash[27868]: cluster 2023-10-09T04:20:05.182564+0000 mgr.y (mgr.34107) 102 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:08.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:08 smithi153 bash[27868]: cluster 2023-10-09T04:20:07.183179+0000 mgr.y (mgr.34107) 103 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:08.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:20:08 smithi153 bash[55216]: ts=2023-10-09T04:20:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:20:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:09] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-09T04:20:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:10 smithi153 bash[27868]: cluster 2023-10-09T04:20:09.183819+0000 mgr.y (mgr.34107) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:12.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:12 smithi153 bash[27868]: cluster 2023-10-09T04:20:11.184911+0000 mgr.y (mgr.34107) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:14 smithi153 bash[27868]: cluster 2023-10-09T04:20:13.185512+0000 mgr.y (mgr.34107) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:15.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:20:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:15 smithi153 bash[27868]: audit 2023-10-09T04:20:14.250832+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:16.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:16 smithi153 bash[27868]: cluster 2023-10-09T04:20:15.186541+0000 mgr.y (mgr.34107) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:18 smithi153 bash[27868]: cluster 2023-10-09T04:20:17.187235+0000 mgr.y (mgr.34107) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:18.664 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:18 smithi153 bash[27868]: audit 2023-10-09T04:20:18.142645+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:18.681 INFO:teuthology.orchestra.run.smithi082.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:18.927 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:20:18 smithi153 bash[55216]: ts=2023-10-09T04:20:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:20:19.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T04:20:19.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:19] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.33.4" 2023-10-09T04:20:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:19 smithi153 bash[27868]: audit 2023-10-09T04:20:18.668707+0000 mgr.y (mgr.34107) 109 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:19 smithi153 bash[27868]: cephadm 2023-10-09T04:20:18.669326+0000 mgr.y (mgr.34107) 110 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:19 smithi153 bash[27868]: audit 2023-10-09T04:20:18.678098+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:21.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:20 smithi153 bash[27868]: cluster 2023-10-09T04:20:19.187837+0000 mgr.y (mgr.34107) 111 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:22.390 INFO:teuthology.orchestra.run.smithi082.stdout:wait for all mons to upgrade 2023-10-09T04:20:23.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:22 smithi153 bash[27868]: cluster 2023-10-09T04:20:21.188883+0000 mgr.y (mgr.34107) 112 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:23.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T04:20:24.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:23 smithi153 bash[27868]: cluster 2023-10-09T04:20:23.189535+0000 mgr.y (mgr.34107) 113 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:24.448178+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:24.458668+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:24.797766+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:24.807820+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.338568+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.339989+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.349651+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:25.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.432444+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:25.754 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.436433+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:25.754 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.438488+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:20:25.754 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:25 smithi153 bash[27868]: audit 2023-10-09T04:20:25.439498+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:20:26.883 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:26 smithi153 bash[27868]: cluster 2023-10-09T04:20:25.190703+0000 mgr.y (mgr.34107) 114 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:26.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:26 smithi153 bash[27868]: cephadm 2023-10-09T04:20:25.440309+0000 mgr.y (mgr.34107) 115 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T04:20:26.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:26 smithi153 bash[27868]: audit 2023-10-09T04:20:26.403041+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.82:0/1013611739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: cluster 2023-10-09T04:20:27.191337+0000 mgr.y (mgr.34107) 116 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: cephadm 2023-10-09T04:20:27.559621+0000 mgr.y (mgr.34107) 117 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: audit 2023-10-09T04:20:27.569107+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: audit 2023-10-09T04:20:27.570698+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: audit 2023-10-09T04:20:27.571974+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: audit 2023-10-09T04:20:27.573164+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:28.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[27868]: cephadm 2023-10-09T04:20:27.574788+0000 mgr.y (mgr.34107) 118 : cephadm [INF] Deploying daemon mon.b on smithi153 2023-10-09T04:20:28.874 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 04:20:28 smithi153 bash[55216]: ts=2023-10-09T04:20:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi153.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.82\", device_class=\"ssd\", hostname=\"smithi082\", instance=\"smithi082:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.82\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:20:29.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:29 smithi153 bash[27868]: audit 2023-10-09T04:20:28.855768+0000 mon.a (mon.0) 149 : audit [DBG] from='client.? 172.21.15.82:0/2145021873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:29.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:29 smithi153 bash[27868]: audit 2023-10-09T04:20:29.245939+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:29.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:29 smithi153 systemd[1]: Stopping Ceph mon.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:20:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:29] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.33.4" 2023-10-09T04:20:30.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:29 smithi153 bash[27868]: debug 2023-10-09T04:20:29.833+0000 7fb65ce8b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T04:20:30.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:29 smithi153 bash[27868]: debug 2023-10-09T04:20:29.833+0000 7fb65ce8b700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:20:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:30 smithi153 bash[59445]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T04:20:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:30 smithi153 bash[59508]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T04:20:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:30 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T04:20:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:30 smithi153 systemd[1]: Stopped Ceph mon.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:20:31.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:31 smithi153 systemd[1]: Started Ceph mon.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:20:32.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:31 smithi153 bash[59565]: debug 2023-10-09T04:20:31.985+0000 7fb98ffb2c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:31 smithi153 bash[59565]: debug 2023-10-09T04:20:31.985+0000 7fb98ffb2c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:31 smithi153 bash[59565]: debug 2023-10-09T04:20:31.985+0000 7fb98ffb2c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.993+0000 7fb98ffb2c80 0 load: jerasure load: lrc 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Git sha 0 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: DB SUMMARY 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: DB Session ID: UJXKZ9DZ3PV3M4QIZHWC 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:20:32.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1459 Bytes 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000029.sst 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000027.log size: 3307439 ; 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.env: 0x564f36fff720 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.info_log: 0x564f38887480 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:20:32.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T04:20:32.180 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T04:20:32.180 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:20:32.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.db_log_dir: 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.wal_dir: 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.write_buffer_manager: 0x564f37b14c80 2023-10-09T04:20:32.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:20:32.183 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.row_cache: None 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.wal_filter: None 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T04:20:32.184 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:20:32.185 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T04:20:32.186 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Compression algorithms supported: 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kZSTD supported: 0 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:20:32.187 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.merge_operator: 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564f38887580) 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: cache_index_and_filter_blocks: 1 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:20:32.188 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: pin_top_level_index_and_filter: 1 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: index_type: 0 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: data_block_index_type: 0 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: index_shortening: 1 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: checksum: 4 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: no_block_cache: 0 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_cache: 0x564f37b971f0 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_cache_name: BinnedLRUCache 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_cache_options: 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: capacity : 536870912 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: num_shard_bits : 4 2023-10-09T04:20:32.189 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: strict_capacity_limit : 0 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: high_pri_pool_ratio: 0.000 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_cache_compressed: (nil) 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: persistent_cache: (nil) 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_size: 4096 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_size_deviation: 10 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_restart_interval: 16 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: index_block_restart_interval: 1 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: metadata_block_size: 4096 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: partition_filters: 0 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: use_delta_encoding: 1 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: filter_policy: bloomfilter 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: whole_key_filtering: 1 2023-10-09T04:20:32.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: verify_compression: 0 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: read_amp_bytes_per_bit: 0 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: format_version: 5 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: enable_index_compression: 1 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: block_align: 0 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: max_auto_readahead_size: 262144 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: prepopulate_block_cache: 0 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: initial_auto_readahead_size: 8192 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: num_file_reads_for_auto_readahead: 2 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T04:20:32.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.num_levels: 7 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:20:32.192 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:20:32.193 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:20:32.194 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:20:32.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:20:32.196 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:20:32.196 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:20:32.197 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:20:32.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb984fcf700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 29.sst 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 31, last_sequence is 18446, log_number is 27,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:20:32.199 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 27 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fcf73627-dcee-4ca6-8f9d-ddd37a485c87 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825232003337, "job": 1, "event": "recovery_started", "wal_files": [27]} 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:31.997+0000 7fb98ffb2c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #27 mode 2 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825232012218, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 32, "file_size": 2196640, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18441, "largest_seqno": 19085, "table_properties": {"data_size": 2192349, "index_size": 2761, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 3846, "raw_average_key_size": 20, "raw_value_size": 2186687, "raw_average_value_size": 11569, "num_data_blocks": 121, "num_entries": 189, "num_filter_entries": 189, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696825232, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fcf73627-dcee-4ca6-8f9d-ddd37a485c87", "db_session_id": "UJXKZ9DZ3PV3M4QIZHWC", "orig_file_number": 32, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696825232012368, "job": 1, "event": "recovery_finished"} 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 34 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000027.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564f37c38000 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 4 rocksdb: DB pointer 0x564f37c26000 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb97e7c2700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb97e7c2700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:20:32.200 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: ** DB Stats ** 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: ** Compaction Stats [default] ** 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: L0 1/0 2.09 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T04:20:32.201 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: L6 1/0 32.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Sum 2/0 34.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: ** Compaction Stats [default] ** 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 549.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:20:32.202 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Cumulative compaction: 0.00 GB write, 167.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Interval compaction: 0.00 GB write, 167.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Block cache BinnedLRUCache@0x564f37b971f0#7 capacity: 512.00 MB usage: 3.42 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.009+0000 7fb98ffb2c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] at bind addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 1 mon.b@-1(???) e3 preinit fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).mds e6 new map 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).mds e6 print_map 2023-10-09T04:20:32.203 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: e6 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: legacy client fscid: 1 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Filesystem 'foo' (1) 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: fs_name foo 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: epoch 5 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: created 2023-10-09T04:11:26.223721+0000 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: modified 2023-10-09T04:11:33.267531+0000 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: tableserver 0 2023-10-09T04:20:32.204 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: root 0 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: session_timeout 60 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: session_autoclose 300 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: max_file_size 1099511627776 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: required_client_features {} 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: last_failure 0 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: last_failure_osd_epoch 0 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: max_mds 1 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: in 0 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: up {0=14640} 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: failed 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: damaged 2023-10-09T04:20:32.205 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: stopped 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: data_pools [4] 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: metadata_pool 3 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: inline_data disabled 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: balancer 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: bal_rank_mask -1 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: standby_count_wanted 1 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: [mds.foo.smithi082.juwitc{0:14640} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.82:6834/3414512881,v1:172.21.15.82:6835/3414512881] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: Standby daemons: 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: 2023-10-09T04:20:32.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: [mds.foo.smithi153.braqax{-1:24481} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.153:6832/2932221885,v1:172.21.15.153:6833/2932221885] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 1 mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T04:20:32.207 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T04:20:32.208 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T04:20:32.208 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T04:20:32.208 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T04:20:32.208 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T04:20:32.208 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:32 smithi153 bash[59565]: debug 2023-10-09T04:20:32.013+0000 7fb98ffb2c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T04:20:37.482 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:37 smithi082 bash[73210]: debug 2023-10-09T04:20:37.023+0000 7fed11da6700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T04:20:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:00:57.352746+0000 mon.a (mon.0) 0 : 1 mkfs 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:20:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:20:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm 1 Deploying daemon mgr.y on smithi082 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm 1 Failing over to other MGR 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm 1 Deploying daemon mgr.x on smithi153 2023-10-09T04:20:39.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm 1 Deploying daemon mon.a on smithi082 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:30.413505+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTING 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:30.536485+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:30.638606+0000 mgr.y (mgr.34107) 5 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:30.638960+0000 mgr.y (mgr.34107) 6 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTED 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:33.915163+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:45.788444+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:45.788754+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:45.953385+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:46.003343+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.680 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:46.118500+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:46.168185+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:46.283521+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:17:46.333106+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:18:41.502341+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:48.415435+0000 mon.a (mon.0) 81 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:18:48.416181+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:49.064580+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/1551051967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:18:50.635539+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T04:20:39.681 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:50.644120+0000 mon.a (mon.0) 82 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.682 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:50.645798+0000 mon.a (mon.0) 83 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.682 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:50.646882+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.682 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:50.647908+0000 mon.a (mon.0) 85 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.682 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:18:50.649235+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi082 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:18:51.505396+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/4073491519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:18:55.590970+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:01.652057+0000 mon.a (mon.0) 106 : cluster 0 mgrmap e34: y(active, since 92s), standbys: x 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:01.652310+0000 mon.a (mon.0) 107 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:01.652346+0000 mon.a (mon.0) 108 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:01.678877+0000 mon.a (mon.0) 109 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:01.698062+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e35: y(active, since 92s), standbys: x 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:02.410861+0000 mon.a (mon.0) 111 : audit 0 from='client.? 172.21.15.82:0/1888849131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:03.157296+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:05.158202+0000 mgr.y (mgr.34107) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:06.424727+0000 mgr.y (mgr.34107) 70 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:06.830735+0000 mon.a (mon.0) 112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:06.838044+0000 mon.a (mon.0) 113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:07.158824+0000 mgr.y (mgr.34107) 71 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:07.625103+0000 mon.a (mon.0) 114 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:07.630811+0000 mon.a (mon.0) 115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:09.159426+0000 mgr.y (mgr.34107) 72 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:10.500707+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:19:10.501366+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:10.507897+0000 mon.a (mon.0) 116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:11.160500+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:13.161146+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:14.243976+0000 mon.a (mon.0) 117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.684 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:15.162126+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:17.162661+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:17.932441+0000 mon.a (mon.0) 118 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:17.943208+0000 mon.a (mon.0) 119 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:17.944936+0000 mon.a (mon.0) 120 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:17.946168+0000 mon.a (mon.0) 121 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:17.952501+0000 mon.a (mon.0) 122 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:18.035277+0000 mon.a (mon.0) 123 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:18.038188+0000 mon.a (mon.0) 124 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:18.039525+0000 mon.a (mon.0) 125 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:18.047060+0000 mon.a (mon.0) 126 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:19.163299+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:21.164353+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:23.164981+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:25.165855+0000 mgr.y (mgr.34107) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:27.166527+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:29.167254+0000 mgr.y (mgr.34107) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:29.244733+0000 mon.a (mon.0) 127 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:31.168344+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:33.168965+0000 mgr.y (mgr.34107) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:35.169915+0000 mgr.y (mgr.34107) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:37.170534+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:39.171056+0000 mgr.y (mgr.34107) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:41.172097+0000 mgr.y (mgr.34107) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.686 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:43.172881+0000 mgr.y (mgr.34107) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:44.244714+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:45.173965+0000 mgr.y (mgr.34107) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:47.174647+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:49.175405+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:51.176573+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:53.177175+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:55.178324+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:57.179042+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:19:59.179774+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:19:59.245329+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:00.000148+0000 mon.a (mon.0) 130 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.687 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:01.180938+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:03.181618+0000 mgr.y (mgr.34107) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:05.182564+0000 mgr.y (mgr.34107) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:07.183179+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:09.183819+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:11.184911+0000 mgr.y (mgr.34107) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:13.185512+0000 mgr.y (mgr.34107) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:14.250832+0000 mon.a (mon.0) 131 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:15.186541+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:17.187235+0000 mgr.y (mgr.34107) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:18.142645+0000 mon.a (mon.0) 132 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:18.668707+0000 mgr.y (mgr.34107) 109 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.688 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:20:18.669326+0000 mgr.y (mgr.34107) 110 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:18.678098+0000 mon.a (mon.0) 133 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:19.187837+0000 mgr.y (mgr.34107) 111 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:21.188883+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:23.189535+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:24.448178+0000 mon.a (mon.0) 134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:24.458668+0000 mon.a (mon.0) 135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:24.797766+0000 mon.a (mon.0) 136 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:24.807820+0000 mon.a (mon.0) 137 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:25.190703+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.338568+0000 mon.a (mon.0) 138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.339989+0000 mon.a (mon.0) 139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.689 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.349651+0000 mon.a (mon.0) 140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.432444+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.436433+0000 mon.a (mon.0) 142 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.438488+0000 mon.a (mon.0) 143 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:25.439498+0000 mon.a (mon.0) 144 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:20:25.440309+0000 mgr.y (mgr.34107) 115 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:26.403041+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.82:0/1013611739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:27.191337+0000 mgr.y (mgr.34107) 116 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:20:27.559621+0000 mgr.y (mgr.34107) 117 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:27.569107+0000 mon.a (mon.0) 145 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:27.570698+0000 mon.a (mon.0) 146 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:27.571974+0000 mon.a (mon.0) 147 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.690 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:27.573164+0000 mon.a (mon.0) 148 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cephadm 2023-10-09T04:20:27.574788+0000 mgr.y (mgr.34107) 118 : cephadm 1 Deploying daemon mon.b on smithi153 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:28.855768+0000 mon.a (mon.0) 149 : audit 0 from='client.? 172.21.15.82:0/2145021873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:29.245939+0000 mon.a (mon.0) 150 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:33.241348+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:33.244220+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.248191+0000 mon.a (mon.0) 163 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.263099+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.269973+0000 mon.a (mon.0) 164 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.270042+0000 mon.a (mon.0) 165 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.270602+0000 mon.a (mon.0) 166 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.270868+0000 mon.a (mon.0) 167 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.273808+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.691 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.274211+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.294589+0000 mon.a (mon.0) 170 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.304706+0000 mon.a (mon.0) 171 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.321918+0000 mon.a (mon.0) 172 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.321977+0000 mon.a (mon.0) 173 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.322628+0000 mon.a (mon.0) 174 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.322917+0000 mon.a (mon.0) 175 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.322960+0000 mon.a (mon.0) 176 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: cluster 2023-10-09T04:20:38.331065+0000 mon.a (mon.0) 177 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:38.335829+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:38.343128+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:39 smithi153 bash[59565]: audit 2023-10-09T04:20:38.368796+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:00:57.352746+0000 mon.a (mon.0) 0 : 1 mkfs 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:20:39.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:20:39.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm 1 Deploying daemon mgr.y on smithi082 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm 1 Failing over to other MGR 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T04:20:39.704 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm 1 Deploying daemon mgr.x on smithi153 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm 1 Deploying daemon mon.a on smithi082 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:30.413505+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTING 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:30.536485+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:30.638606+0000 mgr.y (mgr.34107) 5 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:30.638960+0000 mgr.y (mgr.34107) 6 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTED 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:33.915163+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:45.788444+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:45.788754+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:45.953385+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:46.003343+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:46.118500+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:46.168185+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:46.283521+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:17:46.333106+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:18:41.502341+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:48.415435+0000 mon.a (mon.0) 81 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:18:48.416181+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:49.064580+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/1551051967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:18:50.635539+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:50.644120+0000 mon.a (mon.0) 82 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:00:57.352746+0000 mon.a (mon.0) 0 : 1 mkfs 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:15:42.114853+0000 mgr.x (mgr.25177) 50 : cephadm 1 Reconfiguring daemon grafana.a on smithi153 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:05.561314+0000 mgr.x (mgr.25177) 65 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:05.561917+0000 mgr.x (mgr.25177) 66 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:07.682066+0000 mgr.x (mgr.25177) 68 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:07.698680+0000 mgr.x (mgr.25177) 69 : cephadm 1 Deploying daemon mgr.y on smithi082 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:27.258166+0000 mgr.x (mgr.25177) 80 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:27.259330+0000 mgr.x (mgr.25177) 81 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:27.264610+0000 mgr.x (mgr.25177) 82 : cephadm 1 Failing over to other MGR 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:28.068440+0000 mgr.y (mgr.15897) 1 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTING 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:28.171168+0000 mgr.y (mgr.15897) 2 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:28.299216+0000 mgr.y (mgr.15897) 3 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:28.299486+0000 mgr.y (mgr.15897) 4 : cephadm 1 [09/Oct/2023:04:16:28] ENGINE Bus STARTED 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.093909+0000 mgr.y (mgr.15897) 14 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.094251+0000 mgr.y (mgr.15897) 15 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.259048+0000 mgr.y (mgr.15897) 16 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.259380+0000 mgr.y (mgr.15897) 17 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.425059+0000 mgr.y (mgr.15897) 18 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.475048+0000 mgr.y (mgr.15897) 19 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.708 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.589740+0000 mgr.y (mgr.15897) 20 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.640855+0000 mgr.y (mgr.15897) 21 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:44.902364+0000 mgr.y (mgr.15897) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:45.149855+0000 mgr.y (mgr.15897) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi153 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:52.170682+0000 mgr.y (mgr.15897) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:16:52.176989+0000 mgr.y (mgr.15897) 30 : cephadm 1 Deploying daemon mgr.x on smithi153 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:50.645798+0000 mon.a (mon.0) 83 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:50.646882+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:50.647908+0000 mon.a (mon.0) 85 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:18:50.649235+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi082 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:18:51.505396+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/4073491519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:18:55.590970+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:01.652057+0000 mon.a (mon.0) 106 : cluster 0 mgrmap e34: y(active, since 92s), standbys: x 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:01.652310+0000 mon.a (mon.0) 107 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:01.652346+0000 mon.a (mon.0) 108 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:01.678877+0000 mon.a (mon.0) 109 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:01.698062+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e35: y(active, since 92s), standbys: x 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:02.410861+0000 mon.a (mon.0) 111 : audit 0 from='client.? 172.21.15.82:0/1888849131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:03.157296+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:05.158202+0000 mgr.y (mgr.34107) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:06.424727+0000 mgr.y (mgr.34107) 70 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:06.830735+0000 mon.a (mon.0) 112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:06.838044+0000 mon.a (mon.0) 113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:07.158824+0000 mgr.y (mgr.34107) 71 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:07.625103+0000 mon.a (mon.0) 114 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:07.630811+0000 mon.a (mon.0) 115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:09.159426+0000 mgr.y (mgr.34107) 72 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:10.500707+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:11.967218+0000 mgr.y (mgr.15897) 41 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:12.002060+0000 mgr.y (mgr.15897) 42 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T04:20:39.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:14.324238+0000 mgr.y (mgr.15897) 44 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:14.339667+0000 mgr.y (mgr.15897) 45 : cephadm 1 Deploying daemon mon.a on smithi082 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:30.413505+0000 mgr.y (mgr.34107) 3 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTING 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:30.536485+0000 mgr.y (mgr.34107) 4 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on https://172.21.15.82:7150 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:30.638606+0000 mgr.y (mgr.34107) 5 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Serving on http://172.21.15.82:8765 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:30.638960+0000 mgr.y (mgr.34107) 6 : cephadm 1 [09/Oct/2023:04:17:30] ENGINE Bus STARTED 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:33.915163+0000 mgr.y (mgr.34107) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:45.788444+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi082:/etc/ceph/ceph.conf 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:45.788754+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi153:/etc/ceph/ceph.conf 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:45.953385+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:46.003343+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:46.118500+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:46.168185+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi153:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:46.283521+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:17:46.333106+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:18:41.502341+0000 mgr.y (mgr.34107) 54 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:48.415435+0000 mon.a (mon.0) 81 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:18:48.416181+0000 mgr.y (mgr.34107) 58 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:49.064580+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/1551051967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:18:50.635539+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:50.644120+0000 mon.a (mon.0) 82 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.713 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:19:10.501366+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:10.507897+0000 mon.a (mon.0) 116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:11.160500+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:13.161146+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:14.243976+0000 mon.a (mon.0) 117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:15.162126+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:17.162661+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:17.932441+0000 mon.a (mon.0) 118 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:17.943208+0000 mon.a (mon.0) 119 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:17.944936+0000 mon.a (mon.0) 120 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:17.946168+0000 mon.a (mon.0) 121 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:17.952501+0000 mon.a (mon.0) 122 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:18.035277+0000 mon.a (mon.0) 123 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:18.038188+0000 mon.a (mon.0) 124 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:18.039525+0000 mon.a (mon.0) 125 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:18.047060+0000 mon.a (mon.0) 126 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:19.163299+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:21.164353+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:23.164981+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:25.165855+0000 mgr.y (mgr.34107) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:50.645798+0000 mon.a (mon.0) 83 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.715 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:50.646882+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:50.647908+0000 mon.a (mon.0) 85 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:18:50.649235+0000 mgr.y (mgr.34107) 61 : cephadm 1 Deploying daemon mon.c on smithi082 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:18:51.505396+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/4073491519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:18:55.590970+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:01.652057+0000 mon.a (mon.0) 106 : cluster 0 mgrmap e34: y(active, since 92s), standbys: x 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:01.652310+0000 mon.a (mon.0) 107 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:01.652346+0000 mon.a (mon.0) 108 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:01.678877+0000 mon.a (mon.0) 109 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:01.698062+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e35: y(active, since 92s), standbys: x 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:02.410861+0000 mon.a (mon.0) 111 : audit 0 from='client.? 172.21.15.82:0/1888849131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:03.157296+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:05.158202+0000 mgr.y (mgr.34107) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:06.424727+0000 mgr.y (mgr.34107) 70 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:06.830735+0000 mon.a (mon.0) 112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:06.838044+0000 mon.a (mon.0) 113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:07.158824+0000 mgr.y (mgr.34107) 71 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:07.625103+0000 mon.a (mon.0) 114 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:07.630811+0000 mon.a (mon.0) 115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:09.159426+0000 mgr.y (mgr.34107) 72 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:10.500707+0000 mgr.y (mgr.34107) 73 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:27.166527+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:29.167254+0000 mgr.y (mgr.34107) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:29.244733+0000 mon.a (mon.0) 127 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:31.168344+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:33.168965+0000 mgr.y (mgr.34107) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:35.169915+0000 mgr.y (mgr.34107) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:37.170534+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:39.171056+0000 mgr.y (mgr.34107) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:41.172097+0000 mgr.y (mgr.34107) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:43.172881+0000 mgr.y (mgr.34107) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:44.244714+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:45.173965+0000 mgr.y (mgr.34107) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:47.174647+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:49.175405+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:51.176573+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:53.177175+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:55.178324+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:57.179042+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:19:59.179774+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:19:59.245329+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:00.000148+0000 mon.a (mon.0) 130 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:19:10.501366+0000 mgr.y (mgr.34107) 74 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:10.507897+0000 mon.a (mon.0) 116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:11.160500+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:13.161146+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:14.243976+0000 mon.a (mon.0) 117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:15.162126+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:17.162661+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:17.932441+0000 mon.a (mon.0) 118 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:17.943208+0000 mon.a (mon.0) 119 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:17.944936+0000 mon.a (mon.0) 120 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:17.946168+0000 mon.a (mon.0) 121 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:17.952501+0000 mon.a (mon.0) 122 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:18.035277+0000 mon.a (mon.0) 123 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:18.038188+0000 mon.a (mon.0) 124 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:18.039525+0000 mon.a (mon.0) 125 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:18.047060+0000 mon.a (mon.0) 126 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:19.163299+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:21.164353+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:23.164981+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:25.165855+0000 mgr.y (mgr.34107) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:01.180938+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:03.181618+0000 mgr.y (mgr.34107) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:05.182564+0000 mgr.y (mgr.34107) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:07.183179+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:09.183819+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:11.184911+0000 mgr.y (mgr.34107) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:13.185512+0000 mgr.y (mgr.34107) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:14.250832+0000 mon.a (mon.0) 131 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:15.186541+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:17.187235+0000 mgr.y (mgr.34107) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:18.142645+0000 mon.a (mon.0) 132 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:18.668707+0000 mgr.y (mgr.34107) 109 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:20:18.669326+0000 mgr.y (mgr.34107) 110 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:18.678098+0000 mon.a (mon.0) 133 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:19.187837+0000 mgr.y (mgr.34107) 111 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:21.188883+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:23.189535+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:24.448178+0000 mon.a (mon.0) 134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:24.458668+0000 mon.a (mon.0) 135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:24.797766+0000 mon.a (mon.0) 136 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:27.166527+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:29.167254+0000 mgr.y (mgr.34107) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:29.244733+0000 mon.a (mon.0) 127 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:31.168344+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:33.168965+0000 mgr.y (mgr.34107) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:35.169915+0000 mgr.y (mgr.34107) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:37.170534+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:39.171056+0000 mgr.y (mgr.34107) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:41.172097+0000 mgr.y (mgr.34107) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:43.172881+0000 mgr.y (mgr.34107) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:44.244714+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:45.173965+0000 mgr.y (mgr.34107) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:47.174647+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:49.175405+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:51.176573+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:53.177175+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:55.178324+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:57.179042+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:19:59.179774+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:19:59.245329+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:00.000148+0000 mon.a (mon.0) 130 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:24.807820+0000 mon.a (mon.0) 137 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:25.190703+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.338568+0000 mon.a (mon.0) 138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.339989+0000 mon.a (mon.0) 139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.349651+0000 mon.a (mon.0) 140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.432444+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.436433+0000 mon.a (mon.0) 142 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.438488+0000 mon.a (mon.0) 143 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:25.439498+0000 mon.a (mon.0) 144 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:20:25.440309+0000 mgr.y (mgr.34107) 115 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:26.403041+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.82:0/1013611739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:27.191337+0000 mgr.y (mgr.34107) 116 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:20:27.559621+0000 mgr.y (mgr.34107) 117 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:27.569107+0000 mon.a (mon.0) 145 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:27.570698+0000 mon.a (mon.0) 146 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:27.571974+0000 mon.a (mon.0) 147 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:27.573164+0000 mon.a (mon.0) 148 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cephadm 2023-10-09T04:20:27.574788+0000 mgr.y (mgr.34107) 118 : cephadm 1 Deploying daemon mon.b on smithi153 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:28.855768+0000 mon.a (mon.0) 149 : audit 0 from='client.? 172.21.15.82:0/2145021873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:29.245939+0000 mon.a (mon.0) 150 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:33.241348+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:33.244220+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:01.180938+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:03.181618+0000 mgr.y (mgr.34107) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.727 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:05.182564+0000 mgr.y (mgr.34107) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:07.183179+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:09.183819+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:11.184911+0000 mgr.y (mgr.34107) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:13.185512+0000 mgr.y (mgr.34107) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:14.250832+0000 mon.a (mon.0) 131 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:15.186541+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:17.187235+0000 mgr.y (mgr.34107) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:18.142645+0000 mon.a (mon.0) 132 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:18.668707+0000 mgr.y (mgr.34107) 109 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:20:18.669326+0000 mgr.y (mgr.34107) 110 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:39.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:18.678098+0000 mon.a (mon.0) 133 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:19.187837+0000 mgr.y (mgr.34107) 111 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:21.188883+0000 mgr.y (mgr.34107) 112 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:23.189535+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:24.448178+0000 mon.a (mon.0) 134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:24.458668+0000 mon.a (mon.0) 135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:24.797766+0000 mon.a (mon.0) 136 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.248191+0000 mon.a (mon.0) 163 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.263099+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.269973+0000 mon.a (mon.0) 164 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.270042+0000 mon.a (mon.0) 165 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.270602+0000 mon.a (mon.0) 166 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.270868+0000 mon.a (mon.0) 167 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.273808+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.274211+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.294589+0000 mon.a (mon.0) 170 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.304706+0000 mon.a (mon.0) 171 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.321918+0000 mon.a (mon.0) 172 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.321977+0000 mon.a (mon.0) 173 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.322628+0000 mon.a (mon.0) 174 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.322917+0000 mon.a (mon.0) 175 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.322960+0000 mon.a (mon.0) 176 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: cluster 2023-10-09T04:20:38.331065+0000 mon.a (mon.0) 177 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:38.335829+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:38.343128+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[77063]: audit 2023-10-09T04:20:38.368796+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:24.807820+0000 mon.a (mon.0) 137 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:25.190703+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.338568+0000 mon.a (mon.0) 138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.339989+0000 mon.a (mon.0) 139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:39.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.349651+0000 mon.a (mon.0) 140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.432444+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.436433+0000 mon.a (mon.0) 142 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.438488+0000 mon.a (mon.0) 143 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:25.439498+0000 mon.a (mon.0) 144 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:20:25.440309+0000 mgr.y (mgr.34107) 115 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:26.403041+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.82:0/1013611739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:27.191337+0000 mgr.y (mgr.34107) 116 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:20:27.559621+0000 mgr.y (mgr.34107) 117 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:27.569107+0000 mon.a (mon.0) 145 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:27.570698+0000 mon.a (mon.0) 146 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:27.571974+0000 mon.a (mon.0) 147 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:27.573164+0000 mon.a (mon.0) 148 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cephadm 2023-10-09T04:20:27.574788+0000 mgr.y (mgr.34107) 118 : cephadm 1 Deploying daemon mon.b on smithi153 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:28.855768+0000 mon.a (mon.0) 149 : audit 0 from='client.? 172.21.15.82:0/2145021873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:29.245939+0000 mon.a (mon.0) 150 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:33.241348+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:33.244220+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.248191+0000 mon.a (mon.0) 163 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.263099+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.269973+0000 mon.a (mon.0) 164 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.270042+0000 mon.a (mon.0) 165 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.270602+0000 mon.a (mon.0) 166 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.270868+0000 mon.a (mon.0) 167 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:20:39.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.273808+0000 mon.a (mon.0) 168 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.274211+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.294589+0000 mon.a (mon.0) 170 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.304706+0000 mon.a (mon.0) 171 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],b=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0],c=[v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]} removed_ranks: {} 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.321918+0000 mon.a (mon.0) 172 : cluster 0 fsmap foo:1 {0=foo.smithi082.juwitc=up:active} 1 up:standby 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.321977+0000 mon.a (mon.0) 173 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.322628+0000 mon.a (mon.0) 174 : cluster 0 mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.322917+0000 mon.a (mon.0) 175 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.322960+0000 mon.a (mon.0) 176 : cluster 1 Cluster is now healthy 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: cluster 2023-10-09T04:20:38.331065+0000 mon.a (mon.0) 177 : cluster 1 overall HEALTH_OK 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:38.335829+0000 mon.a (mon.0) 178 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:38.343128+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:39.735 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[79733]: audit 2023-10-09T04:20:38.368796+0000 mon.a (mon.0) 180 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:39.778 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:20:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:39] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.33.4" 2023-10-09T04:20:40.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:40 smithi082 bash[77063]: cluster 2023-10-09T04:20:39.195907+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:40.641 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:40 smithi082 bash[77063]: audit 2023-10-09T04:20:39.758076+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.82:0/2945588781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:40.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:40 smithi082 bash[79733]: cluster 2023-10-09T04:20:39.195907+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:40.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:40 smithi082 bash[79733]: audit 2023-10-09T04:20:39.758076+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.82:0/2945588781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:40.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:20:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:40 smithi153 bash[59565]: cluster 2023-10-09T04:20:39.195907+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:40.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:40 smithi153 bash[59565]: audit 2023-10-09T04:20:39.758076+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.82:0/2945588781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:20:42.664 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:42 smithi082 bash[77063]: cluster 2023-10-09T04:20:41.197000+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:42.664 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:42 smithi082 bash[79733]: cluster 2023-10-09T04:20:41.197000+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:42 smithi153 bash[59565]: cluster 2023-10-09T04:20:41.197000+0000 mgr.y (mgr.34107) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:43.944 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (5m) - 11m 13.4M - 0.23.0 ba2b418f427c 4994b766acec 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (4m) - 10m 40.7M - 8.3.5 dad864ee21e9 24b93cfba910 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (9m) - 9m 16.3M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (9m) - 9m 13.7M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443,9283,8765 running (3m) - 17m 427M - 18.2.0-598-gb4fa663f 3060897fdf97 b4d6262d244c 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:8443,9283,8765 running (4m) - 19m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 588466e4b0b0 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (3m) - 19m 51.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 93a348625d31 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 starting - - - 2048M 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 running (109s) - 18m 35.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 27025ca926cb 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (5m) - 11m 8771k - 1.3.1 1dbe0e931976 cda947bde3d9 2023-10-09T04:20:43.945 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (5m) - 11m 8932k - 1.3.1 1dbe0e931976 93816ab22ea5 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (17m) - 17m 45.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9b8a7733f2dc 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (16m) - 16m 47.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 97a8f958f136 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (15m) - 15m 44.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7af0967827d4 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (15m) - 15m 43.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 50ddb6f18074 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (14m) - 14m 47.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (13m) - 13m 44.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (12m) - 12m 45.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (12m) - 12m 49.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:20:43.946 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (3m) - 11m 56.1M - 2.33.4 514e6a882f6e be2d59b8aa70 2023-10-09T04:20:44.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:44 smithi082 bash[77063]: cluster 2023-10-09T04:20:43.197721+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:44.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:44 smithi082 bash[77063]: audit 2023-10-09T04:20:44.246199+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:44 smithi082 bash[79733]: cluster 2023-10-09T04:20:43.197721+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:44.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:44 smithi082 bash[79733]: audit 2023-10-09T04:20:44.246199+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:44 smithi153 bash[59565]: cluster 2023-10-09T04:20:43.197721+0000 mgr.y (mgr.34107) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:44.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:44 smithi153 bash[59565]: audit 2023-10-09T04:20:44.246199+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:20:45.871 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[77063]: audit 2023-10-09T04:20:43.928992+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:45.871 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[77063]: audit 2023-10-09T04:20:44.617761+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.871 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[77063]: audit 2023-10-09T04:20:44.627862+0000 mon.a (mon.0) 183 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[77063]: audit 2023-10-09T04:20:45.422136+0000 mon.a (mon.0) 184 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[77063]: audit 2023-10-09T04:20:45.432295+0000 mon.a (mon.0) 185 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[79733]: audit 2023-10-09T04:20:43.928992+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[79733]: audit 2023-10-09T04:20:44.617761+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[79733]: audit 2023-10-09T04:20:44.627862+0000 mon.a (mon.0) 183 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[79733]: audit 2023-10-09T04:20:45.422136+0000 mon.a (mon.0) 184 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.872 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:45 smithi082 bash[79733]: audit 2023-10-09T04:20:45.432295+0000 mon.a (mon.0) 185 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:45 smithi153 bash[59565]: audit 2023-10-09T04:20:43.928992+0000 mgr.y (mgr.34107) 127 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:45.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:45 smithi153 bash[59565]: audit 2023-10-09T04:20:44.617761+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:45 smithi153 bash[59565]: audit 2023-10-09T04:20:44.627862+0000 mon.a (mon.0) 183 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:45 smithi153 bash[59565]: audit 2023-10-09T04:20:45.422136+0000 mon.a (mon.0) 184 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:45.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:45 smithi153 bash[59565]: audit 2023-10-09T04:20:45.432295+0000 mon.a (mon.0) 185 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:46.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:46 smithi153 bash[59565]: cluster 2023-10-09T04:20:45.198406+0000 mgr.y (mgr.34107) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:46 smithi082 bash[77063]: cluster 2023-10-09T04:20:45.198406+0000 mgr.y (mgr.34107) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:46 smithi082 bash[79733]: cluster 2023-10-09T04:20:45.198406+0000 mgr.y (mgr.34107) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:48.114 INFO:teuthology.orchestra.run.smithi082.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:48.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:20:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:48 smithi153 bash[59565]: cluster 2023-10-09T04:20:47.198978+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:48 smithi153 bash[59565]: audit 2023-10-09T04:20:48.111586+0000 mon.a (mon.0) 186 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:48 smithi082 bash[77063]: cluster 2023-10-09T04:20:47.198978+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:48.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:48 smithi082 bash[77063]: audit 2023-10-09T04:20:48.111586+0000 mon.a (mon.0) 186 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:48 smithi082 bash[79733]: cluster 2023-10-09T04:20:47.198978+0000 mgr.y (mgr.34107) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:48 smithi082 bash[79733]: audit 2023-10-09T04:20:48.111586+0000 mon.a (mon.0) 186 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:49.922 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:49] "GET /metrics HTTP/1.1" 200 33762 "" "Prometheus/2.33.4" 2023-10-09T04:20:49.922 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:49 smithi082 bash[77063]: audit 2023-10-09T04:20:48.102886+0000 mgr.y (mgr.34107) 130 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:49.923 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:49 smithi082 bash[77063]: cephadm 2023-10-09T04:20:48.103497+0000 mgr.y (mgr.34107) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:49.923 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:49 smithi082 bash[79733]: audit 2023-10-09T04:20:48.102886+0000 mgr.y (mgr.34107) 130 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:49.923 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:49 smithi082 bash[79733]: cephadm 2023-10-09T04:20:48.103497+0000 mgr.y (mgr.34107) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:49.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:49 smithi153 bash[59565]: audit 2023-10-09T04:20:48.102886+0000 mgr.y (mgr.34107) 130 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:20:49.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:49 smithi153 bash[59565]: cephadm 2023-10-09T04:20:48.103497+0000 mgr.y (mgr.34107) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:20:50.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:50 smithi153 bash[59565]: cluster 2023-10-09T04:20:49.199504+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:50 smithi082 bash[77063]: cluster 2023-10-09T04:20:49.199504+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:50 smithi082 bash[79733]: cluster 2023-10-09T04:20:49.199504+0000 mgr.y (mgr.34107) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:52.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:52 smithi153 bash[59565]: cluster 2023-10-09T04:20:51.200554+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:52.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:52 smithi082 bash[77063]: cluster 2023-10-09T04:20:51.200554+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:52.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:52 smithi082 bash[79733]: cluster 2023-10-09T04:20:51.200554+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: cluster 2023-10-09T04:20:53.201178+0000 mgr.y (mgr.34107) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: audit 2023-10-09T04:20:54.586030+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: audit 2023-10-09T04:20:54.596999+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: audit 2023-10-09T04:20:54.598975+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: audit 2023-10-09T04:20:54.600366+0000 mon.a (mon.0) 190 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:54 smithi153 bash[59565]: audit 2023-10-09T04:20:54.610131+0000 mon.a (mon.0) 191 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: cluster 2023-10-09T04:20:53.201178+0000 mgr.y (mgr.34107) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: audit 2023-10-09T04:20:54.586030+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: audit 2023-10-09T04:20:54.596999+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: audit 2023-10-09T04:20:54.598975+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: audit 2023-10-09T04:20:54.600366+0000 mon.a (mon.0) 190 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[77063]: audit 2023-10-09T04:20:54.610131+0000 mon.a (mon.0) 191 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: cluster 2023-10-09T04:20:53.201178+0000 mgr.y (mgr.34107) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: audit 2023-10-09T04:20:54.586030+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: audit 2023-10-09T04:20:54.596999+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: audit 2023-10-09T04:20:54.598975+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: audit 2023-10-09T04:20:54.600366+0000 mon.a (mon.0) 190 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:54 smithi082 bash[79733]: audit 2023-10-09T04:20:54.610131+0000 mon.a (mon.0) 191 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:55.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[79733]: cephadm 2023-10-09T04:20:54.680315+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:20:55.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[79733]: audit 2023-10-09T04:20:54.680917+0000 mon.a (mon.0) 192 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:55.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[79733]: audit 2023-10-09T04:20:54.682494+0000 mon.a (mon.0) 193 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:55.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[79733]: audit 2023-10-09T04:20:54.683980+0000 mon.a (mon.0) 194 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:55.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[79733]: cephadm 2023-10-09T04:20:54.685719+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi082 2023-10-09T04:20:55.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[77063]: cephadm 2023-10-09T04:20:54.680315+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:20:55.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[77063]: audit 2023-10-09T04:20:54.680917+0000 mon.a (mon.0) 192 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:55.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[77063]: audit 2023-10-09T04:20:54.682494+0000 mon.a (mon.0) 193 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:55.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[77063]: audit 2023-10-09T04:20:54.683980+0000 mon.a (mon.0) 194 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:55.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:55 smithi082 bash[77063]: cephadm 2023-10-09T04:20:54.685719+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi082 2023-10-09T04:20:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:55 smithi153 bash[59565]: cephadm 2023-10-09T04:20:54.680315+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T04:20:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:55 smithi153 bash[59565]: audit 2023-10-09T04:20:54.680917+0000 mon.a (mon.0) 192 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:55 smithi153 bash[59565]: audit 2023-10-09T04:20:54.682494+0000 mon.a (mon.0) 193 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:55 smithi153 bash[59565]: audit 2023-10-09T04:20:54.683980+0000 mon.a (mon.0) 194 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:55 smithi153 bash[59565]: cephadm 2023-10-09T04:20:54.685719+0000 mgr.y (mgr.34107) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi082 2023-10-09T04:20:56.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:56 smithi153 bash[59565]: cluster 2023-10-09T04:20:55.202147+0000 mgr.y (mgr.34107) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:56.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:56 smithi082 bash[77063]: cluster 2023-10-09T04:20:55.202147+0000 mgr.y (mgr.34107) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:56.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:56 smithi082 bash[79733]: cluster 2023-10-09T04:20:55.202147+0000 mgr.y (mgr.34107) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:58.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: audit 2023-10-09T04:20:56.754340+0000 mon.a (mon.0) 195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: audit 2023-10-09T04:20:56.761328+0000 mon.a (mon.0) 196 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: cephadm 2023-10-09T04:20:56.803993+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:20:58.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: audit 2023-10-09T04:20:56.804800+0000 mon.a (mon.0) 197 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:20:58.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: audit 2023-10-09T04:20:56.806898+0000 mon.a (mon.0) 198 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: audit 2023-10-09T04:20:56.808584+0000 mon.a (mon.0) 199 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: cephadm 2023-10-09T04:20:56.810198+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[79733]: cluster 2023-10-09T04:20:57.202794+0000 mgr.y (mgr.34107) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: audit 2023-10-09T04:20:56.754340+0000 mon.a (mon.0) 195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: audit 2023-10-09T04:20:56.761328+0000 mon.a (mon.0) 196 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: cephadm 2023-10-09T04:20:56.803993+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: audit 2023-10-09T04:20:56.804800+0000 mon.a (mon.0) 197 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: audit 2023-10-09T04:20:56.806898+0000 mon.a (mon.0) 198 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:20:58.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: audit 2023-10-09T04:20:56.808584+0000 mon.a (mon.0) 199 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:58.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: cephadm 2023-10-09T04:20:56.810198+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:20:58.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:57 smithi082 bash[77063]: cluster 2023-10-09T04:20:57.202794+0000 mgr.y (mgr.34107) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: audit 2023-10-09T04:20:56.754340+0000 mon.a (mon.0) 195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: audit 2023-10-09T04:20:56.761328+0000 mon.a (mon.0) 196 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: cephadm 2023-10-09T04:20:56.803993+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: audit 2023-10-09T04:20:56.804800+0000 mon.a (mon.0) 197 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: audit 2023-10-09T04:20:56.806898+0000 mon.a (mon.0) 198 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:20:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: audit 2023-10-09T04:20:56.808584+0000 mon.a (mon.0) 199 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:58.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: cephadm 2023-10-09T04:20:56.810198+0000 mgr.y (mgr.34107) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi082 2023-10-09T04:20:58.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:57 smithi153 bash[59565]: cluster 2023-10-09T04:20:57.202794+0000 mgr.y (mgr.34107) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:59.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:20:59] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:20:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:58.855752+0000 mon.a (mon.0) 200 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:58.863631+0000 mon.a (mon.0) 201 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: cephadm 2023-10-09T04:20:58.879502+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:20:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:58.880099+0000 mon.a (mon.0) 202 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:59.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:58.881671+0000 mon.a (mon.0) 203 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:59.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:58.883174+0000 mon.a (mon.0) 204 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:59.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: cephadm 2023-10-09T04:20:58.884997+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi082 2023-10-09T04:20:59.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: cluster 2023-10-09T04:20:59.203218+0000 mgr.y (mgr.34107) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:59.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[77063]: audit 2023-10-09T04:20:59.246501+0000 mon.a (mon.0) 205 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:58.855752+0000 mon.a (mon.0) 200 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:58.863631+0000 mon.a (mon.0) 201 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: cephadm 2023-10-09T04:20:58.879502+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:58.880099+0000 mon.a (mon.0) 202 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:58.881671+0000 mon.a (mon.0) 203 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:58.883174+0000 mon.a (mon.0) 204 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: cephadm 2023-10-09T04:20:58.884997+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi082 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: cluster 2023-10-09T04:20:59.203218+0000 mgr.y (mgr.34107) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:20:59.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:20:59 smithi082 bash[79733]: audit 2023-10-09T04:20:59.246501+0000 mon.a (mon.0) 205 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:58.855752+0000 mon.a (mon.0) 200 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:58.863631+0000 mon.a (mon.0) 201 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: cephadm 2023-10-09T04:20:58.879502+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:58.880099+0000 mon.a (mon.0) 202 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:58.881671+0000 mon.a (mon.0) 203 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:58.883174+0000 mon.a (mon.0) 204 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: cephadm 2023-10-09T04:20:58.884997+0000 mgr.y (mgr.34107) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi082 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: cluster 2023-10-09T04:20:59.203218+0000 mgr.y (mgr.34107) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:20:59 smithi153 bash[59565]: audit 2023-10-09T04:20:59.246501+0000 mon.a (mon.0) 205 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:02.225 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: audit 2023-10-09T04:21:00.953651+0000 mon.a (mon.0) 206 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: audit 2023-10-09T04:21:00.961137+0000 mon.a (mon.0) 207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: cephadm 2023-10-09T04:21:01.008701+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:21:02.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: audit 2023-10-09T04:21:01.009371+0000 mon.a (mon.0) 208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:21:02.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: audit 2023-10-09T04:21:01.011096+0000 mon.a (mon.0) 209 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:02.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: cephadm 2023-10-09T04:21:01.014708+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi082 2023-10-09T04:21:02.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[77063]: cluster 2023-10-09T04:21:01.204241+0000 mgr.y (mgr.34107) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:02.228 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: audit 2023-10-09T04:21:00.953651+0000 mon.a (mon.0) 206 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.228 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: audit 2023-10-09T04:21:00.961137+0000 mon.a (mon.0) 207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: cephadm 2023-10-09T04:21:01.008701+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:21:02.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: audit 2023-10-09T04:21:01.009371+0000 mon.a (mon.0) 208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:21:02.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: audit 2023-10-09T04:21:01.011096+0000 mon.a (mon.0) 209 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:02.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: cephadm 2023-10-09T04:21:01.014708+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi082 2023-10-09T04:21:02.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:01 smithi082 bash[79733]: cluster 2023-10-09T04:21:01.204241+0000 mgr.y (mgr.34107) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: audit 2023-10-09T04:21:00.953651+0000 mon.a (mon.0) 206 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: audit 2023-10-09T04:21:00.961137+0000 mon.a (mon.0) 207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: cephadm 2023-10-09T04:21:01.008701+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: audit 2023-10-09T04:21:01.009371+0000 mon.a (mon.0) 208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: audit 2023-10-09T04:21:01.011096+0000 mon.a (mon.0) 209 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: cephadm 2023-10-09T04:21:01.014708+0000 mgr.y (mgr.34107) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi082 2023-10-09T04:21:02.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:01 smithi153 bash[59565]: cluster 2023-10-09T04:21:01.204241+0000 mgr.y (mgr.34107) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:04.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: audit 2023-10-09T04:21:03.090190+0000 mon.a (mon.0) 210 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: audit 2023-10-09T04:21:03.099206+0000 mon.a (mon.0) 211 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: cephadm 2023-10-09T04:21:03.133743+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:21:04.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: audit 2023-10-09T04:21:03.134336+0000 mon.a (mon.0) 212 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:21:04.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: audit 2023-10-09T04:21:03.136020+0000 mon.a (mon.0) 213 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:04.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: cephadm 2023-10-09T04:21:03.140436+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi082 2023-10-09T04:21:04.428 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:04 smithi153 bash[59565]: cluster 2023-10-09T04:21:03.204918+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: audit 2023-10-09T04:21:03.090190+0000 mon.a (mon.0) 210 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: audit 2023-10-09T04:21:03.099206+0000 mon.a (mon.0) 211 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: cephadm 2023-10-09T04:21:03.133743+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: audit 2023-10-09T04:21:03.134336+0000 mon.a (mon.0) 212 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: audit 2023-10-09T04:21:03.136020+0000 mon.a (mon.0) 213 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: cephadm 2023-10-09T04:21:03.140436+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi082 2023-10-09T04:21:04.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[77063]: cluster 2023-10-09T04:21:03.204918+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: audit 2023-10-09T04:21:03.090190+0000 mon.a (mon.0) 210 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: audit 2023-10-09T04:21:03.099206+0000 mon.a (mon.0) 211 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: cephadm 2023-10-09T04:21:03.133743+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: audit 2023-10-09T04:21:03.134336+0000 mon.a (mon.0) 212 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: audit 2023-10-09T04:21:03.136020+0000 mon.a (mon.0) 213 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: cephadm 2023-10-09T04:21:03.140436+0000 mgr.y (mgr.34107) 148 : cephadm 1 Reconfiguring daemon osd.1 on smithi082 2023-10-09T04:21:04.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:04 smithi082 bash[79733]: cluster 2023-10-09T04:21:03.204918+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: cluster 2023-10-09T04:21:05.206048+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: audit 2023-10-09T04:21:05.222629+0000 mon.a (mon.0) 214 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: audit 2023-10-09T04:21:05.229438+0000 mon.a (mon.0) 215 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: cephadm 2023-10-09T04:21:05.258096+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: audit 2023-10-09T04:21:05.258810+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: audit 2023-10-09T04:21:05.264600+0000 mon.a (mon.0) 217 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[77063]: cephadm 2023-10-09T04:21:05.268345+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi082 2023-10-09T04:21:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: cluster 2023-10-09T04:21:05.206048+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: audit 2023-10-09T04:21:05.222629+0000 mon.a (mon.0) 214 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: audit 2023-10-09T04:21:05.229438+0000 mon.a (mon.0) 215 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: cephadm 2023-10-09T04:21:05.258096+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: audit 2023-10-09T04:21:05.258810+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: audit 2023-10-09T04:21:05.264600+0000 mon.a (mon.0) 217 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:06 smithi082 bash[79733]: cephadm 2023-10-09T04:21:05.268345+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi082 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: cluster 2023-10-09T04:21:05.206048+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: audit 2023-10-09T04:21:05.222629+0000 mon.a (mon.0) 214 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: audit 2023-10-09T04:21:05.229438+0000 mon.a (mon.0) 215 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: cephadm 2023-10-09T04:21:05.258096+0000 mgr.y (mgr.34107) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: audit 2023-10-09T04:21:05.258810+0000 mon.a (mon.0) 216 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: audit 2023-10-09T04:21:05.264600+0000 mon.a (mon.0) 217 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:06 smithi153 bash[59565]: cephadm 2023-10-09T04:21:05.268345+0000 mgr.y (mgr.34107) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi082 2023-10-09T04:21:08.626 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: cluster 2023-10-09T04:21:07.206750+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:08.626 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: audit 2023-10-09T04:21:07.320476+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.626 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: audit 2023-10-09T04:21:07.328188+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.626 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: cephadm 2023-10-09T04:21:07.336560+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: audit 2023-10-09T04:21:07.337048+0000 mon.a (mon.0) 220 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: audit 2023-10-09T04:21:07.338454+0000 mon.a (mon.0) 221 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[77063]: cephadm 2023-10-09T04:21:07.342144+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi082 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: cluster 2023-10-09T04:21:07.206750+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: audit 2023-10-09T04:21:07.320476+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: audit 2023-10-09T04:21:07.328188+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: cephadm 2023-10-09T04:21:07.336560+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:21:08.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: audit 2023-10-09T04:21:07.337048+0000 mon.a (mon.0) 220 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:21:08.628 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: audit 2023-10-09T04:21:07.338454+0000 mon.a (mon.0) 221 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:08.628 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:08 smithi082 bash[79733]: cephadm 2023-10-09T04:21:07.342144+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi082 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: cluster 2023-10-09T04:21:07.206750+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: audit 2023-10-09T04:21:07.320476+0000 mon.a (mon.0) 218 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: audit 2023-10-09T04:21:07.328188+0000 mon.a (mon.0) 219 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: cephadm 2023-10-09T04:21:07.336560+0000 mgr.y (mgr.34107) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: audit 2023-10-09T04:21:07.337048+0000 mon.a (mon.0) 220 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: audit 2023-10-09T04:21:07.338454+0000 mon.a (mon.0) 221 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:08 smithi153 bash[59565]: cephadm 2023-10-09T04:21:07.342144+0000 mgr.y (mgr.34107) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi082 2023-10-09T04:21:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:09] "GET /metrics HTTP/1.1" 200 33773 "" "Prometheus/2.33.4" 2023-10-09T04:21:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: cluster 2023-10-09T04:21:09.207400+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: audit 2023-10-09T04:21:09.415049+0000 mon.a (mon.0) 222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: audit 2023-10-09T04:21:09.424435+0000 mon.a (mon.0) 223 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: cephadm 2023-10-09T04:21:09.461620+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mds.foo.smithi082.juwitc (monmap changed)... 2023-10-09T04:21:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: audit 2023-10-09T04:21:09.462265+0000 mon.a (mon.0) 224 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:10.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: audit 2023-10-09T04:21:09.463967+0000 mon.a (mon.0) 225 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:10.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:10 smithi153 bash[59565]: cephadm 2023-10-09T04:21:09.465723+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:21:10.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: cluster 2023-10-09T04:21:09.207400+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:10.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: audit 2023-10-09T04:21:09.415049+0000 mon.a (mon.0) 222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: audit 2023-10-09T04:21:09.424435+0000 mon.a (mon.0) 223 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: cephadm 2023-10-09T04:21:09.461620+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mds.foo.smithi082.juwitc (monmap changed)... 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: audit 2023-10-09T04:21:09.462265+0000 mon.a (mon.0) 224 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: audit 2023-10-09T04:21:09.463967+0000 mon.a (mon.0) 225 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[77063]: cephadm 2023-10-09T04:21:09.465723+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: cluster 2023-10-09T04:21:09.207400+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: audit 2023-10-09T04:21:09.415049+0000 mon.a (mon.0) 222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: audit 2023-10-09T04:21:09.424435+0000 mon.a (mon.0) 223 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:10.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: cephadm 2023-10-09T04:21:09.461620+0000 mgr.y (mgr.34107) 157 : cephadm 1 Reconfiguring mds.foo.smithi082.juwitc (monmap changed)... 2023-10-09T04:21:10.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: audit 2023-10-09T04:21:09.462265+0000 mon.a (mon.0) 224 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi082.juwitc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:10.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: audit 2023-10-09T04:21:09.463967+0000 mon.a (mon.0) 225 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:10.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:10 smithi082 bash[79733]: cephadm 2023-10-09T04:21:09.465723+0000 mgr.y (mgr.34107) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi082.juwitc on smithi082 2023-10-09T04:21:12.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: cluster 2023-10-09T04:21:11.208091+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:12.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: audit 2023-10-09T04:21:11.506999+0000 mon.a (mon.0) 226 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: audit 2023-10-09T04:21:11.514203+0000 mon.a (mon.0) 227 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: cephadm 2023-10-09T04:21:11.535395+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:21:12.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: audit 2023-10-09T04:21:11.535972+0000 mon.a (mon.0) 228 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:21:12.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: audit 2023-10-09T04:21:11.537572+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:21:12.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: audit 2023-10-09T04:21:11.539091+0000 mon.a (mon.0) 230 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:12.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:12 smithi153 bash[59565]: cephadm 2023-10-09T04:21:11.540876+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi153 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: cluster 2023-10-09T04:21:11.208091+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: audit 2023-10-09T04:21:11.506999+0000 mon.a (mon.0) 226 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: audit 2023-10-09T04:21:11.514203+0000 mon.a (mon.0) 227 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: cephadm 2023-10-09T04:21:11.535395+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: audit 2023-10-09T04:21:11.535972+0000 mon.a (mon.0) 228 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:21:12.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: audit 2023-10-09T04:21:11.537572+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: audit 2023-10-09T04:21:11.539091+0000 mon.a (mon.0) 230 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[77063]: cephadm 2023-10-09T04:21:11.540876+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi153 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: cluster 2023-10-09T04:21:11.208091+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: audit 2023-10-09T04:21:11.506999+0000 mon.a (mon.0) 226 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: audit 2023-10-09T04:21:11.514203+0000 mon.a (mon.0) 227 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:12.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: cephadm 2023-10-09T04:21:11.535395+0000 mgr.y (mgr.34107) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T04:21:12.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: audit 2023-10-09T04:21:11.535972+0000 mon.a (mon.0) 228 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:21:12.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: audit 2023-10-09T04:21:11.537572+0000 mon.a (mon.0) 229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:21:12.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: audit 2023-10-09T04:21:11.539091+0000 mon.a (mon.0) 230 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:12.985 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:12 smithi082 bash[79733]: cephadm 2023-10-09T04:21:11.540876+0000 mgr.y (mgr.34107) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi153 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: cluster 2023-10-09T04:21:13.208744+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:13.519067+0000 mon.a (mon.0) 231 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:13.526744+0000 mon.a (mon.0) 232 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: cephadm 2023-10-09T04:21:13.560252+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:13.560936+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:13.562839+0000 mon.a (mon.0) 234 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:13.564364+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:14.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: cephadm 2023-10-09T04:21:13.566182+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi153 2023-10-09T04:21:14.862 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:14 smithi153 bash[59565]: audit 2023-10-09T04:21:14.247238+0000 mon.a (mon.0) 236 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: cluster 2023-10-09T04:21:13.208744+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:13.519067+0000 mon.a (mon.0) 231 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:13.526744+0000 mon.a (mon.0) 232 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: cephadm 2023-10-09T04:21:13.560252+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:13.560936+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:13.562839+0000 mon.a (mon.0) 234 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:13.564364+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: cephadm 2023-10-09T04:21:13.566182+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi153 2023-10-09T04:21:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[77063]: audit 2023-10-09T04:21:14.247238+0000 mon.a (mon.0) 236 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: cluster 2023-10-09T04:21:13.208744+0000 mgr.y (mgr.34107) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:13.519067+0000 mon.a (mon.0) 231 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:13.526744+0000 mon.a (mon.0) 232 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: cephadm 2023-10-09T04:21:13.560252+0000 mgr.y (mgr.34107) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:13.560936+0000 mon.a (mon.0) 233 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:13.562839+0000 mon.a (mon.0) 234 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:13.564364+0000 mon.a (mon.0) 235 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: cephadm 2023-10-09T04:21:13.566182+0000 mgr.y (mgr.34107) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi153 2023-10-09T04:21:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:14 smithi082 bash[79733]: audit 2023-10-09T04:21:14.247238+0000 mon.a (mon.0) 236 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: cluster 2023-10-09T04:21:15.209795+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: audit 2023-10-09T04:21:15.593517+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: audit 2023-10-09T04:21:15.601314+0000 mon.a (mon.0) 238 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: cephadm 2023-10-09T04:21:15.636833+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: audit 2023-10-09T04:21:15.637445+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: audit 2023-10-09T04:21:15.638985+0000 mon.a (mon.0) 240 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:16.866 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:16 smithi153 bash[59565]: cephadm 2023-10-09T04:21:15.642739+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi153 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: cluster 2023-10-09T04:21:15.209795+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: audit 2023-10-09T04:21:15.593517+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: audit 2023-10-09T04:21:15.601314+0000 mon.a (mon.0) 238 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: cephadm 2023-10-09T04:21:15.636833+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: audit 2023-10-09T04:21:15.637445+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: audit 2023-10-09T04:21:15.638985+0000 mon.a (mon.0) 240 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:16.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[77063]: cephadm 2023-10-09T04:21:15.642739+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi153 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: cluster 2023-10-09T04:21:15.209795+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: audit 2023-10-09T04:21:15.593517+0000 mon.a (mon.0) 237 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: audit 2023-10-09T04:21:15.601314+0000 mon.a (mon.0) 238 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: cephadm 2023-10-09T04:21:15.636833+0000 mgr.y (mgr.34107) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: audit 2023-10-09T04:21:15.637445+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: audit 2023-10-09T04:21:15.638985+0000 mon.a (mon.0) 240 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:16.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:16 smithi082 bash[79733]: cephadm 2023-10-09T04:21:15.642739+0000 mgr.y (mgr.34107) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi153 2023-10-09T04:21:18.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: cluster 2023-10-09T04:21:17.210414+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:18.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: audit 2023-10-09T04:21:17.580736+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: audit 2023-10-09T04:21:17.588759+0000 mon.a (mon.0) 242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.902 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: cephadm 2023-10-09T04:21:17.611157+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:21:18.902 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: audit 2023-10-09T04:21:17.611724+0000 mon.a (mon.0) 243 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:21:18.902 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: audit 2023-10-09T04:21:17.613426+0000 mon.a (mon.0) 244 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:18.902 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:18 smithi153 bash[59565]: cephadm 2023-10-09T04:21:17.617720+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.5 on smithi153 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: cluster 2023-10-09T04:21:17.210414+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: audit 2023-10-09T04:21:17.580736+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: audit 2023-10-09T04:21:17.588759+0000 mon.a (mon.0) 242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: cephadm 2023-10-09T04:21:17.611157+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: audit 2023-10-09T04:21:17.611724+0000 mon.a (mon.0) 243 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: audit 2023-10-09T04:21:17.613426+0000 mon.a (mon.0) 244 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:18.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[77063]: cephadm 2023-10-09T04:21:17.617720+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.5 on smithi153 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: cluster 2023-10-09T04:21:17.210414+0000 mgr.y (mgr.34107) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: audit 2023-10-09T04:21:17.580736+0000 mon.a (mon.0) 241 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: audit 2023-10-09T04:21:17.588759+0000 mon.a (mon.0) 242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: cephadm 2023-10-09T04:21:17.611157+0000 mgr.y (mgr.34107) 169 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: audit 2023-10-09T04:21:17.611724+0000 mon.a (mon.0) 243 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: audit 2023-10-09T04:21:17.613426+0000 mon.a (mon.0) 244 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:18.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:18 smithi082 bash[79733]: cephadm 2023-10-09T04:21:17.617720+0000 mgr.y (mgr.34107) 170 : cephadm 1 Reconfiguring daemon osd.5 on smithi153 2023-10-09T04:21:19.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:19] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.33.4" 2023-10-09T04:21:20.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: cluster 2023-10-09T04:21:19.211099+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:20.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: audit 2023-10-09T04:21:19.608621+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: audit 2023-10-09T04:21:19.618601+0000 mon.a (mon.0) 246 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: cephadm 2023-10-09T04:21:19.636465+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:21:20.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: audit 2023-10-09T04:21:19.637104+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:21:20.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: audit 2023-10-09T04:21:19.638791+0000 mon.a (mon.0) 248 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:20.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:20 smithi153 bash[59565]: cephadm 2023-10-09T04:21:19.642579+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.6 on smithi153 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: cluster 2023-10-09T04:21:19.211099+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: audit 2023-10-09T04:21:19.608621+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: audit 2023-10-09T04:21:19.618601+0000 mon.a (mon.0) 246 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: cephadm 2023-10-09T04:21:19.636465+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: audit 2023-10-09T04:21:19.637104+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: audit 2023-10-09T04:21:19.638791+0000 mon.a (mon.0) 248 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[77063]: cephadm 2023-10-09T04:21:19.642579+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.6 on smithi153 2023-10-09T04:21:20.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: cluster 2023-10-09T04:21:19.211099+0000 mgr.y (mgr.34107) 171 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: audit 2023-10-09T04:21:19.608621+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: audit 2023-10-09T04:21:19.618601+0000 mon.a (mon.0) 246 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: cephadm 2023-10-09T04:21:19.636465+0000 mgr.y (mgr.34107) 172 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: audit 2023-10-09T04:21:19.637104+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: audit 2023-10-09T04:21:19.638791+0000 mon.a (mon.0) 248 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:20.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:20 smithi082 bash[79733]: cephadm 2023-10-09T04:21:19.642579+0000 mgr.y (mgr.34107) 173 : cephadm 1 Reconfiguring daemon osd.6 on smithi153 2023-10-09T04:21:22.893 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: cluster 2023-10-09T04:21:21.212171+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: audit 2023-10-09T04:21:21.634281+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: audit 2023-10-09T04:21:21.641861+0000 mon.a (mon.0) 250 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: cephadm 2023-10-09T04:21:21.661325+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: audit 2023-10-09T04:21:21.662031+0000 mon.a (mon.0) 251 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: audit 2023-10-09T04:21:21.663903+0000 mon.a (mon.0) 252 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:22.894 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:22 smithi153 bash[59565]: cephadm 2023-10-09T04:21:21.668547+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon osd.7 on smithi153 2023-10-09T04:21:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: cluster 2023-10-09T04:21:21.212171+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: audit 2023-10-09T04:21:21.634281+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: audit 2023-10-09T04:21:21.641861+0000 mon.a (mon.0) 250 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: cephadm 2023-10-09T04:21:21.661325+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: audit 2023-10-09T04:21:21.662031+0000 mon.a (mon.0) 251 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: audit 2023-10-09T04:21:21.663903+0000 mon.a (mon.0) 252 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[79733]: cephadm 2023-10-09T04:21:21.668547+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon osd.7 on smithi153 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: cluster 2023-10-09T04:21:21.212171+0000 mgr.y (mgr.34107) 174 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:22.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: audit 2023-10-09T04:21:21.634281+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: audit 2023-10-09T04:21:21.641861+0000 mon.a (mon.0) 250 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:22.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: cephadm 2023-10-09T04:21:21.661325+0000 mgr.y (mgr.34107) 175 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:21:22.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: audit 2023-10-09T04:21:21.662031+0000 mon.a (mon.0) 251 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:21:22.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: audit 2023-10-09T04:21:21.663903+0000 mon.a (mon.0) 252 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:22.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:22 smithi082 bash[77063]: cephadm 2023-10-09T04:21:21.668547+0000 mgr.y (mgr.34107) 176 : cephadm 1 Reconfiguring daemon osd.7 on smithi153 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: cluster 2023-10-09T04:21:23.212793+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: audit 2023-10-09T04:21:23.626583+0000 mon.a (mon.0) 253 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: audit 2023-10-09T04:21:23.636094+0000 mon.a (mon.0) 254 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: cephadm 2023-10-09T04:21:23.688995+0000 mgr.y (mgr.34107) 178 : cephadm 1 Reconfiguring mds.foo.smithi153.braqax (monmap changed)... 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: audit 2023-10-09T04:21:23.689781+0000 mon.a (mon.0) 255 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: audit 2023-10-09T04:21:23.691835+0000 mon.a (mon.0) 256 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:24 smithi153 bash[59565]: cephadm 2023-10-09T04:21:23.693654+0000 mgr.y (mgr.34107) 179 : cephadm 1 Reconfiguring daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:21:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: cluster 2023-10-09T04:21:23.212793+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: audit 2023-10-09T04:21:23.626583+0000 mon.a (mon.0) 253 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: audit 2023-10-09T04:21:23.636094+0000 mon.a (mon.0) 254 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: cephadm 2023-10-09T04:21:23.688995+0000 mgr.y (mgr.34107) 178 : cephadm 1 Reconfiguring mds.foo.smithi153.braqax (monmap changed)... 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: audit 2023-10-09T04:21:23.689781+0000 mon.a (mon.0) 255 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: audit 2023-10-09T04:21:23.691835+0000 mon.a (mon.0) 256 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[77063]: cephadm 2023-10-09T04:21:23.693654+0000 mgr.y (mgr.34107) 179 : cephadm 1 Reconfiguring daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: cluster 2023-10-09T04:21:23.212793+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: audit 2023-10-09T04:21:23.626583+0000 mon.a (mon.0) 253 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: audit 2023-10-09T04:21:23.636094+0000 mon.a (mon.0) 254 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:24.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: cephadm 2023-10-09T04:21:23.688995+0000 mgr.y (mgr.34107) 178 : cephadm 1 Reconfiguring mds.foo.smithi153.braqax (monmap changed)... 2023-10-09T04:21:24.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: audit 2023-10-09T04:21:23.689781+0000 mon.a (mon.0) 255 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi153.braqax", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:21:24.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: audit 2023-10-09T04:21:23.691835+0000 mon.a (mon.0) 256 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:24 smithi082 bash[79733]: cephadm 2023-10-09T04:21:23.693654+0000 mgr.y (mgr.34107) 179 : cephadm 1 Reconfiguring daemon mds.foo.smithi153.braqax on smithi153 2023-10-09T04:21:26.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:26 smithi153 bash[59565]: cluster 2023-10-09T04:21:25.213789+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:26.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:26 smithi153 bash[59565]: audit 2023-10-09T04:21:25.673210+0000 mon.a (mon.0) 257 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:26 smithi153 bash[59565]: audit 2023-10-09T04:21:25.684454+0000 mon.a (mon.0) 258 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:26 smithi153 bash[59565]: audit 2023-10-09T04:21:25.726568+0000 mon.a (mon.0) 259 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[77063]: cluster 2023-10-09T04:21:25.213789+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[77063]: audit 2023-10-09T04:21:25.673210+0000 mon.a (mon.0) 257 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[77063]: audit 2023-10-09T04:21:25.684454+0000 mon.a (mon.0) 258 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[77063]: audit 2023-10-09T04:21:25.726568+0000 mon.a (mon.0) 259 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[79733]: cluster 2023-10-09T04:21:25.213789+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[79733]: audit 2023-10-09T04:21:25.673210+0000 mon.a (mon.0) 257 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[79733]: audit 2023-10-09T04:21:25.684454+0000 mon.a (mon.0) 258 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:26 smithi082 bash[79733]: audit 2023-10-09T04:21:25.726568+0000 mon.a (mon.0) 259 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:28.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:28 smithi082 bash[77063]: cluster 2023-10-09T04:21:27.214540+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:28 smithi082 bash[79733]: cluster 2023-10-09T04:21:27.214540+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:29.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:28 smithi153 bash[59565]: cluster 2023-10-09T04:21:27.214540+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:29] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.33.4" 2023-10-09T04:21:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:30 smithi082 bash[77063]: cluster 2023-10-09T04:21:29.215281+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:30 smithi082 bash[77063]: audit 2023-10-09T04:21:29.247259+0000 mon.a (mon.0) 260 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:30 smithi082 bash[79733]: cluster 2023-10-09T04:21:29.215281+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:30 smithi082 bash[79733]: audit 2023-10-09T04:21:29.247259+0000 mon.a (mon.0) 260 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:30 smithi153 bash[59565]: cluster 2023-10-09T04:21:29.215281+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:30 smithi153 bash[59565]: audit 2023-10-09T04:21:29.247259+0000 mon.a (mon.0) 260 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: cluster 2023-10-09T04:21:31.216084+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: audit 2023-10-09T04:21:31.772088+0000 mon.a (mon.0) 261 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: audit 2023-10-09T04:21:31.780762+0000 mon.a (mon.0) 262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: audit 2023-10-09T04:21:32.254182+0000 mon.a (mon.0) 263 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: audit 2023-10-09T04:21:32.255991+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:32 smithi153 bash[59565]: audit 2023-10-09T04:21:32.264728+0000 mon.a (mon.0) 265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: cluster 2023-10-09T04:21:31.216084+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: audit 2023-10-09T04:21:31.772088+0000 mon.a (mon.0) 261 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: audit 2023-10-09T04:21:31.780762+0000 mon.a (mon.0) 262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: audit 2023-10-09T04:21:32.254182+0000 mon.a (mon.0) 263 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: audit 2023-10-09T04:21:32.255991+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:32.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[79733]: audit 2023-10-09T04:21:32.264728+0000 mon.a (mon.0) 265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: cluster 2023-10-09T04:21:31.216084+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: audit 2023-10-09T04:21:31.772088+0000 mon.a (mon.0) 261 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: audit 2023-10-09T04:21:31.780762+0000 mon.a (mon.0) 262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: audit 2023-10-09T04:21:32.254182+0000 mon.a (mon.0) 263 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: audit 2023-10-09T04:21:32.255991+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:32.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:32 smithi082 bash[77063]: audit 2023-10-09T04:21:32.264728+0000 mon.a (mon.0) 265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:34.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:34 smithi153 bash[59565]: cluster 2023-10-09T04:21:33.216667+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:34 smithi082 bash[77063]: cluster 2023-10-09T04:21:33.216667+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:34.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:34 smithi082 bash[79733]: cluster 2023-10-09T04:21:33.216667+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:36.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:36 smithi153 bash[59565]: cluster 2023-10-09T04:21:35.217695+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:36 smithi082 bash[77063]: cluster 2023-10-09T04:21:35.217695+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:36.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:36 smithi082 bash[79733]: cluster 2023-10-09T04:21:35.217695+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:38 smithi153 bash[59565]: cluster 2023-10-09T04:21:37.218408+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:38.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:38 smithi082 bash[77063]: cluster 2023-10-09T04:21:37.218408+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:38.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:38 smithi082 bash[79733]: cluster 2023-10-09T04:21:37.218408+0000 mgr.y (mgr.34107) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:39] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.33.4" 2023-10-09T04:21:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:40 smithi082 bash[77063]: cluster 2023-10-09T04:21:39.219045+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:40.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:40 smithi082 bash[79733]: cluster 2023-10-09T04:21:39.219045+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:40.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:40 smithi153 bash[59565]: cluster 2023-10-09T04:21:39.219045+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:42.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:42 smithi153 bash[59565]: cluster 2023-10-09T04:21:41.220117+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:42.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:42 smithi082 bash[77063]: cluster 2023-10-09T04:21:41.220117+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:42 smithi082 bash[79733]: cluster 2023-10-09T04:21:41.220117+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:44.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:44 smithi153 bash[59565]: cluster 2023-10-09T04:21:43.220924+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:44.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:44 smithi153 bash[59565]: audit 2023-10-09T04:21:44.248035+0000 mon.a (mon.0) 266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:44 smithi082 bash[77063]: cluster 2023-10-09T04:21:43.220924+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:44 smithi082 bash[77063]: audit 2023-10-09T04:21:44.248035+0000 mon.a (mon.0) 266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:44 smithi082 bash[79733]: cluster 2023-10-09T04:21:43.220924+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:44 smithi082 bash[79733]: audit 2023-10-09T04:21:44.248035+0000 mon.a (mon.0) 266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:46 smithi153 bash[59565]: cluster 2023-10-09T04:21:45.221923+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:46 smithi082 bash[77063]: cluster 2023-10-09T04:21:45.221923+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:46.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:46 smithi082 bash[79733]: cluster 2023-10-09T04:21:45.221923+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:48 smithi153 bash[59565]: cluster 2023-10-09T04:21:47.222525+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:48 smithi082 bash[77063]: cluster 2023-10-09T04:21:47.222525+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:48 smithi082 bash[79733]: cluster 2023-10-09T04:21:47.222525+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:49] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.33.4" 2023-10-09T04:21:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:50 smithi082 bash[77063]: cluster 2023-10-09T04:21:49.223127+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:50.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:50 smithi082 bash[79733]: cluster 2023-10-09T04:21:49.223127+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:50.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:50 smithi153 bash[59565]: cluster 2023-10-09T04:21:49.223127+0000 mgr.y (mgr.34107) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:52.675 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:52 smithi082 bash[77063]: cluster 2023-10-09T04:21:51.224163+0000 mgr.y (mgr.34107) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:52.676 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:52 smithi082 bash[79733]: cluster 2023-10-09T04:21:51.224163+0000 mgr.y (mgr.34107) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:52.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:21:52.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:52 smithi153 bash[59565]: cluster 2023-10-09T04:21:51.224163+0000 mgr.y (mgr.34107) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:54.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:54 smithi082 bash[77063]: cluster 2023-10-09T04:21:53.224842+0000 mgr.y (mgr.34107) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:54.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:54 smithi082 bash[79733]: cluster 2023-10-09T04:21:53.224842+0000 mgr.y (mgr.34107) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:54 smithi153 bash[59565]: cluster 2023-10-09T04:21:53.224842+0000 mgr.y (mgr.34107) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:56.075 INFO:teuthology.orchestra.run.smithi082.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:21:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[77063]: cluster 2023-10-09T04:21:55.225758+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[77063]: audit 2023-10-09T04:21:56.072371+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[77063]: audit 2023-10-09T04:21:56.075254+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:56.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[79733]: cluster 2023-10-09T04:21:55.225758+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:56.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[79733]: audit 2023-10-09T04:21:56.072371+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:56.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:56 smithi082 bash[79733]: audit 2023-10-09T04:21:56.075254+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:56 smithi153 bash[59565]: cluster 2023-10-09T04:21:55.225758+0000 mgr.y (mgr.34107) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:56.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:56 smithi153 bash[59565]: audit 2023-10-09T04:21:56.072371+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:21:56.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:56 smithi153 bash[59565]: audit 2023-10-09T04:21:56.075254+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:56.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T04:21:57.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:57 smithi082 bash[77063]: audit 2023-10-09T04:21:56.063869+0000 mgr.y (mgr.34107) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:21:57.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:57 smithi082 bash[77063]: cephadm 2023-10-09T04:21:56.064454+0000 mgr.y (mgr.34107) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:21:57.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:57 smithi082 bash[79733]: audit 2023-10-09T04:21:56.063869+0000 mgr.y (mgr.34107) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:21:57.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:57 smithi082 bash[79733]: cephadm 2023-10-09T04:21:56.064454+0000 mgr.y (mgr.34107) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:21:57.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:57 smithi153 bash[59565]: audit 2023-10-09T04:21:56.063869+0000 mgr.y (mgr.34107) 196 : audit 0 from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:21:57.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:57 smithi153 bash[59565]: cephadm 2023-10-09T04:21:56.064454+0000 mgr.y (mgr.34107) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:21:58.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:21:58 smithi153 bash[59565]: cluster 2023-10-09T04:21:57.226368+0000 mgr.y (mgr.34107) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:58.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:21:58 smithi082 bash[77063]: cluster 2023-10-09T04:21:57.226368+0000 mgr.y (mgr.34107) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:58.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:21:58 smithi082 bash[79733]: cluster 2023-10-09T04:21:57.226368+0000 mgr.y (mgr.34107) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:21:59.801 INFO:teuthology.orchestra.run.smithi082.stdout:wait for half of osds to upgrade 2023-10-09T04:21:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:21:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:21:59] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:22:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:00 smithi082 bash[79733]: cluster 2023-10-09T04:21:59.226991+0000 mgr.y (mgr.34107) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:00 smithi082 bash[79733]: audit 2023-10-09T04:21:59.248285+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:00.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:00 smithi082 bash[77063]: cluster 2023-10-09T04:21:59.226991+0000 mgr.y (mgr.34107) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:00.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:00 smithi082 bash[77063]: audit 2023-10-09T04:21:59.248285+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:00.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T04:22:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:00 smithi153 bash[59565]: cluster 2023-10-09T04:21:59.226991+0000 mgr.y (mgr.34107) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:01.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:00 smithi153 bash[59565]: audit 2023-10-09T04:21:59.248285+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:02 smithi153 bash[59565]: cluster 2023-10-09T04:22:01.228040+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:03.096 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:02 smithi082 bash[77063]: cluster 2023-10-09T04:22:01.228040+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:03.096 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:02 smithi082 bash[79733]: cluster 2023-10-09T04:22:01.228040+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.316805+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.325728+0000 mon.a (mon.0) 271 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.796906+0000 mon.a (mon.0) 272 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.797992+0000 mon.a (mon.0) 273 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.806227+0000 mon.a (mon.0) 274 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.888098+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.892564+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.895216+0000 mon.a (mon.0) 277 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.903926+0000 mon.a (mon.0) 278 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.905540+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.909997+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.911556+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.917175+0000 mon.a (mon.0) 282 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.918432+0000 mon.a (mon.0) 283 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:22:03.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.924046+0000 mon.a (mon.0) 284 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.926147+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.934814+0000 mon.a (mon.0) 286 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[79733]: audit 2023-10-09T04:22:02.936717+0000 mon.a (mon.0) 287 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.316805+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.325728+0000 mon.a (mon.0) 271 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.796906+0000 mon.a (mon.0) 272 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.797992+0000 mon.a (mon.0) 273 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.806227+0000 mon.a (mon.0) 274 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.888098+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.892564+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.895216+0000 mon.a (mon.0) 277 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.903926+0000 mon.a (mon.0) 278 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.905540+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.909997+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.911556+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.917175+0000 mon.a (mon.0) 282 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.918432+0000 mon.a (mon.0) 283 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.924046+0000 mon.a (mon.0) 284 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:22:03.486 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.926147+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.487 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.934814+0000 mon.a (mon.0) 286 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.487 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:03 smithi082 bash[77063]: audit 2023-10-09T04:22:02.936717+0000 mon.a (mon.0) 287 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:03.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.316805+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.325728+0000 mon.a (mon.0) 271 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.796906+0000 mon.a (mon.0) 272 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:03.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.797992+0000 mon.a (mon.0) 273 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.806227+0000 mon.a (mon.0) 274 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.888098+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.892564+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.895216+0000 mon.a (mon.0) 277 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.903926+0000 mon.a (mon.0) 278 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.905540+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.909997+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.911556+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.917175+0000 mon.a (mon.0) 282 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.918432+0000 mon.a (mon.0) 283 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:22:03.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.924046+0000 mon.a (mon.0) 284 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:22:03.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.926147+0000 mon.a (mon.0) 285 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:03.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.934814+0000 mon.a (mon.0) 286 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:03.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:03 smithi153 bash[59565]: audit 2023-10-09T04:22:02.936717+0000 mon.a (mon.0) 287 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: cephadm 2023-10-09T04:22:02.896224+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: cephadm 2023-10-09T04:22:02.927195+0000 mgr.y (mgr.34107) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: audit 2023-10-09T04:22:02.936964+0000 mgr.y (mgr.34107) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: cephadm 2023-10-09T04:22:02.938379+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: cluster 2023-10-09T04:22:03.228557+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:04.517 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[79733]: audit 2023-10-09T04:22:04.095921+0000 mon.a (mon.0) 288 : audit 0 from='client.? 172.21.15.82:0/754205605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: cephadm 2023-10-09T04:22:02.896224+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: cephadm 2023-10-09T04:22:02.927195+0000 mgr.y (mgr.34107) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: audit 2023-10-09T04:22:02.936964+0000 mgr.y (mgr.34107) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: cephadm 2023-10-09T04:22:02.938379+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: cluster 2023-10-09T04:22:03.228557+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:04.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:04 smithi082 bash[77063]: audit 2023-10-09T04:22:04.095921+0000 mon.a (mon.0) 288 : audit 0 from='client.? 172.21.15.82:0/754205605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: cephadm 2023-10-09T04:22:02.896224+0000 mgr.y (mgr.34107) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: cephadm 2023-10-09T04:22:02.927195+0000 mgr.y (mgr.34107) 202 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: audit 2023-10-09T04:22:02.936964+0000 mgr.y (mgr.34107) 203 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: cephadm 2023-10-09T04:22:02.938379+0000 mgr.y (mgr.34107) 204 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: cluster 2023-10-09T04:22:03.228557+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:04.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:04 smithi153 bash[59565]: audit 2023-10-09T04:22:04.095921+0000 mon.a (mon.0) 288 : audit 0 from='client.? 172.21.15.82:0/754205605' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: cephadm 2023-10-09T04:22:05.207588+0000 mgr.y (mgr.34107) 206 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: audit 2023-10-09T04:22:05.215819+0000 mon.a (mon.0) 289 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: audit 2023-10-09T04:22:05.217683+0000 mon.a (mon.0) 290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: audit 2023-10-09T04:22:05.218776+0000 mon.a (mon.0) 291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: cephadm 2023-10-09T04:22:05.221024+0000 mgr.y (mgr.34107) 207 : cephadm 1 Deploying daemon osd.0 on smithi082 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[77063]: cluster 2023-10-09T04:22:05.229471+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: cephadm 2023-10-09T04:22:05.207588+0000 mgr.y (mgr.34107) 206 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: audit 2023-10-09T04:22:05.215819+0000 mon.a (mon.0) 289 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: audit 2023-10-09T04:22:05.217683+0000 mon.a (mon.0) 290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:22:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: audit 2023-10-09T04:22:05.218776+0000 mon.a (mon.0) 291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: cephadm 2023-10-09T04:22:05.221024+0000 mgr.y (mgr.34107) 207 : cephadm 1 Deploying daemon osd.0 on smithi082 2023-10-09T04:22:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:06 smithi082 bash[79733]: cluster 2023-10-09T04:22:05.229471+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: cephadm 2023-10-09T04:22:05.207588+0000 mgr.y (mgr.34107) 206 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: audit 2023-10-09T04:22:05.215819+0000 mon.a (mon.0) 289 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: audit 2023-10-09T04:22:05.217683+0000 mon.a (mon.0) 290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: audit 2023-10-09T04:22:05.218776+0000 mon.a (mon.0) 291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: cephadm 2023-10-09T04:22:05.221024+0000 mgr.y (mgr.34107) 207 : cephadm 1 Deploying daemon osd.0 on smithi082 2023-10-09T04:22:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:06 smithi153 bash[59565]: cluster 2023-10-09T04:22:05.229471+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:07 smithi082 bash[79733]: audit 2023-10-09T04:22:06.455136+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.82:0/2215084876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:07.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:07 smithi082 bash[77063]: audit 2023-10-09T04:22:06.455136+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.82:0/2215084876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:07.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:07 smithi153 bash[59565]: audit 2023-10-09T04:22:06.455136+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.82:0/2215084876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:08.469 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:08 smithi082 bash[77063]: cluster 2023-10-09T04:22:07.229964+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:08.470 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:08 smithi082 bash[79733]: cluster 2023-10-09T04:22:07.229964+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:08.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:08 smithi153 bash[59565]: cluster 2023-10-09T04:22:07.229964+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:09.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[79733]: audit 2023-10-09T04:22:08.801480+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.82:0/2623154796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[77063]: audit 2023-10-09T04:22:08.801480+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.82:0/2623154796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:09 smithi153 bash[59565]: audit 2023-10-09T04:22:08.801480+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.82:0/2623154796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:09] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.33.4" 2023-10-09T04:22:09.982 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:09 smithi082 systemd[1]: Stopping Ceph osd.0 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:22:09.983 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[37060]: debug 2023-10-09T04:22:09.746+0000 7f1c569f0700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:22:09.983 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[37060]: debug 2023-10-09T04:22:09.746+0000 7f1c569f0700 -1 osd.0 84 *** Got signal Terminated *** 2023-10-09T04:22:09.983 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:09 smithi082 bash[37060]: debug 2023-10-09T04:22:09.746+0000 7f1c569f0700 -1 osd.0 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:22:10.501 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:10 smithi082 bash[77063]: cluster 2023-10-09T04:22:09.230667+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:10.501 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:10 smithi082 bash[77063]: cluster 2023-10-09T04:22:09.751566+0000 mon.a (mon.0) 292 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:22:10.501 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:10 smithi082 bash[79733]: cluster 2023-10-09T04:22:09.230667+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:10.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:10 smithi082 bash[79733]: cluster 2023-10-09T04:22:09.751566+0000 mon.a (mon.0) 292 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:22:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:10 smithi153 bash[59565]: cluster 2023-10-09T04:22:09.230667+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:10 smithi153 bash[59565]: cluster 2023-10-09T04:22:09.751566+0000 mon.a (mon.0) 292 : cluster 1 osd.0 marked itself down and dead 2023-10-09T04:22:11.250 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:10 smithi082 bash[85034]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T04:22:11.250 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[85127]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T04:22:11.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:11 smithi153 bash[59565]: cluster 2023-10-09T04:22:10.230272+0000 mon.a (mon.0) 293 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:11.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:11 smithi153 bash[59565]: cluster 2023-10-09T04:22:10.240172+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:11 smithi153 bash[59565]: cluster 2023-10-09T04:22:10.242700+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:11 smithi153 bash[59565]: cluster 2023-10-09T04:22:10.246042+0000 mon.a (mon.0) 294 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:22:11.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:11 smithi153 bash[59565]: audit 2023-10-09T04:22:11.151703+0000 mon.a (mon.0) 295 : audit 0 from='client.? 172.21.15.82:0/677546689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:11.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[77063]: cluster 2023-10-09T04:22:10.230272+0000 mon.a (mon.0) 293 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[77063]: cluster 2023-10-09T04:22:10.240172+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[77063]: cluster 2023-10-09T04:22:10.242700+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[77063]: cluster 2023-10-09T04:22:10.246042+0000 mon.a (mon.0) 294 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[77063]: audit 2023-10-09T04:22:11.151703+0000 mon.a (mon.0) 295 : audit 0 from='client.? 172.21.15.82:0/677546689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[79733]: cluster 2023-10-09T04:22:10.230272+0000 mon.a (mon.0) 293 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[79733]: cluster 2023-10-09T04:22:10.240172+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[79733]: cluster 2023-10-09T04:22:10.242700+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[79733]: cluster 2023-10-09T04:22:10.246042+0000 mon.a (mon.0) 294 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T04:22:11.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:11 smithi082 bash[79733]: audit 2023-10-09T04:22:11.151703+0000 mon.a (mon.0) 295 : audit 0 from='client.? 172.21.15.82:0/677546689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:12.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.231145+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:12.492 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.247104+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.492 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.247620+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250283+0000 mon.a (mon.0) 296 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250441+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250499+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250521+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250603+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.493 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250734+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.494 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250838+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.494 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[77063]: cluster 2023-10-09T04:22:11.250862+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.231145+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.247104+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.247620+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250283+0000 mon.a (mon.0) 296 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250441+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250499+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250521+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250603+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250734+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250838+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.496 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:12 smithi082 bash[79733]: cluster 2023-10-09T04:22:11.250862+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.231145+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:12.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.247104+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.247620+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250283+0000 mon.a (mon.0) 296 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250441+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250499+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250521+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250603+0000 osd.4 (osd.4) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250734+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250838+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:12.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:12 smithi153 bash[59565]: cluster 2023-10-09T04:22:11.250862+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T04:22:13.704 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:13 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T04:22:13.704 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:13 smithi082 systemd[1]: Stopped Ceph osd.0 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:22:13.704 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:13 smithi082 systemd[1]: Started Ceph osd.0 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:22:14.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: cluster 2023-10-09T04:22:13.231552+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:14.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: audit 2023-10-09T04:22:13.546043+0000 mon.a (mon.0) 297 : audit 0 from='client.? 172.21.15.82:0/878921126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:14.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: audit 2023-10-09T04:22:13.677061+0000 mon.a (mon.0) 298 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: audit 2023-10-09T04:22:13.685007+0000 mon.a (mon.0) 299 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: audit 2023-10-09T04:22:13.704404+0000 mon.a (mon.0) 300 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[77063]: audit 2023-10-09T04:22:14.251487+0000 mon.a (mon.0) 301 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: cluster 2023-10-09T04:22:13.231552+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: audit 2023-10-09T04:22:13.546043+0000 mon.a (mon.0) 297 : audit 0 from='client.? 172.21.15.82:0/878921126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: audit 2023-10-09T04:22:13.677061+0000 mon.a (mon.0) 298 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: audit 2023-10-09T04:22:13.685007+0000 mon.a (mon.0) 299 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: audit 2023-10-09T04:22:13.704404+0000 mon.a (mon.0) 300 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:14.528 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:14 smithi082 bash[79733]: audit 2023-10-09T04:22:14.251487+0000 mon.a (mon.0) 301 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: cluster 2023-10-09T04:22:13.231552+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: audit 2023-10-09T04:22:13.546043+0000 mon.a (mon.0) 297 : audit 0 from='client.? 172.21.15.82:0/878921126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: audit 2023-10-09T04:22:13.677061+0000 mon.a (mon.0) 298 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: audit 2023-10-09T04:22:13.685007+0000 mon.a (mon.0) 299 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: audit 2023-10-09T04:22:13.704404+0000 mon.a (mon.0) 300 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:14.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:14 smithi153 bash[59565]: audit 2023-10-09T04:22:14.251487+0000 mon.a (mon.0) 301 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[77063]: audit 2023-10-09T04:22:14.252568+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[79733]: audit 2023-10-09T04:22:14.252568+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:22:15.483 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T04:22:15.484 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:22:15.484 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:15 smithi082 bash[85313]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T04:22:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:15 smithi153 bash[59565]: audit 2023-10-09T04:22:14.252568+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:16 smithi153 bash[59565]: cluster 2023-10-09T04:22:15.232771+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:16.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:16 smithi153 bash[59565]: cluster 2023-10-09T04:22:15.254004+0000 mon.a (mon.0) 303 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:16.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:16 smithi153 bash[59565]: audit 2023-10-09T04:22:15.914601+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.82:0/2682041694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:16.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[77063]: cluster 2023-10-09T04:22:15.232771+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:16.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[77063]: cluster 2023-10-09T04:22:15.254004+0000 mon.a (mon.0) 303 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:16.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[77063]: audit 2023-10-09T04:22:15.914601+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.82:0/2682041694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:16.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[79733]: cluster 2023-10-09T04:22:15.232771+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v148: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:16.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[79733]: cluster 2023-10-09T04:22:15.254004+0000 mon.a (mon.0) 303 : cluster 3 Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:16.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[79733]: audit 2023-10-09T04:22:15.914601+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.82:0/2682041694' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:17.232 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:16 smithi082 bash[85611]: debug 2023-10-09T04:22:16.822+0000 7fddf6a1f700 1 -- 172.21.15.82:0/1354974986 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a7ba5004e0 con 0x55a7ba55a800 2023-10-09T04:22:17.732 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:17 smithi082 bash[85611]: debug 2023-10-09T04:22:17.378+0000 7fddffba5640 -1 Falling back to public interface 2023-10-09T04:22:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:18 smithi153 bash[59565]: cluster 2023-10-09T04:22:17.233591+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:18.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:18 smithi082 bash[79733]: cluster 2023-10-09T04:22:17.233591+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:18.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:18 smithi082 bash[77063]: cluster 2023-10-09T04:22:17.233591+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v149: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:19 smithi153 bash[59565]: audit 2023-10-09T04:22:18.287978+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.82:0/1038399768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:19.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:19 smithi082 bash[77063]: audit 2023-10-09T04:22:18.287978+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.82:0/1038399768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:19 smithi082 bash[79733]: audit 2023-10-09T04:22:18.287978+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.82:0/1038399768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:19] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.33.4" 2023-10-09T04:22:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:20 smithi153 bash[59565]: cluster 2023-10-09T04:22:19.234282+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:20.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:20 smithi153 bash[59565]: audit 2023-10-09T04:22:19.751269+0000 mon.a (mon.0) 304 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:20.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:20 smithi153 bash[59565]: audit 2023-10-09T04:22:19.758452+0000 mon.a (mon.0) 305 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:20.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[77063]: cluster 2023-10-09T04:22:19.234282+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:20.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[77063]: audit 2023-10-09T04:22:19.751269+0000 mon.a (mon.0) 304 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:20.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[77063]: audit 2023-10-09T04:22:19.758452+0000 mon.a (mon.0) 305 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:20.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[79733]: cluster 2023-10-09T04:22:19.234282+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v150: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:20.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[79733]: audit 2023-10-09T04:22:19.751269+0000 mon.a (mon.0) 304 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:20.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[79733]: audit 2023-10-09T04:22:19.758452+0000 mon.a (mon.0) 305 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.232 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:20 smithi082 bash[85611]: debug 2023-10-09T04:22:20.974+0000 7fddffba5640 -1 osd.0 84 log_to_monitors true 2023-10-09T04:22:21.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:21 smithi153 bash[59565]: audit 2023-10-09T04:22:20.556554+0000 mon.a (mon.0) 306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:21 smithi153 bash[59565]: audit 2023-10-09T04:22:20.566025+0000 mon.a (mon.0) 307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:21 smithi153 bash[59565]: audit 2023-10-09T04:22:20.656568+0000 mon.a (mon.0) 308 : audit 0 from='client.? 172.21.15.82:0/1600183098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:21.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:21 smithi153 bash[59565]: audit 2023-10-09T04:22:20.982793+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:22:21.982 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[85611]: debug 2023-10-09T04:22:21.602+0000 7fddee80c700 -1 osd.0 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[77063]: audit 2023-10-09T04:22:20.556554+0000 mon.a (mon.0) 306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[77063]: audit 2023-10-09T04:22:20.566025+0000 mon.a (mon.0) 307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[77063]: audit 2023-10-09T04:22:20.656568+0000 mon.a (mon.0) 308 : audit 0 from='client.? 172.21.15.82:0/1600183098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[77063]: audit 2023-10-09T04:22:20.982793+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[79733]: audit 2023-10-09T04:22:20.556554+0000 mon.a (mon.0) 306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[79733]: audit 2023-10-09T04:22:20.566025+0000 mon.a (mon.0) 307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[79733]: audit 2023-10-09T04:22:20.656568+0000 mon.a (mon.0) 308 : audit 0 from='client.? 172.21.15.82:0/1600183098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:21.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:21 smithi082 bash[79733]: audit 2023-10-09T04:22:20.982793+0000 mon.a (mon.0) 309 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:22:22.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: cluster 2023-10-09T04:22:21.235378+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:22.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: cluster 2023-10-09T04:22:21.569909+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: cluster 2023-10-09T04:22:21.570716+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: audit 2023-10-09T04:22:21.572627+0000 mon.a (mon.0) 310 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:22:22.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: cluster 2023-10-09T04:22:21.572709+0000 mon.a (mon.0) 311 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:22:22.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:22 smithi153 bash[59565]: audit 2023-10-09T04:22:21.573625+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:22.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: cluster 2023-10-09T04:22:21.235378+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:22.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: cluster 2023-10-09T04:22:21.569909+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: cluster 2023-10-09T04:22:21.570716+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: audit 2023-10-09T04:22:21.572627+0000 mon.a (mon.0) 310 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: cluster 2023-10-09T04:22:21.572709+0000 mon.a (mon.0) 311 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[77063]: audit 2023-10-09T04:22:21.573625+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: cluster 2023-10-09T04:22:21.235378+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v151: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: cluster 2023-10-09T04:22:21.569909+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: cluster 2023-10-09T04:22:21.570716+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:22.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: audit 2023-10-09T04:22:21.572627+0000 mon.a (mon.0) 310 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:22:22.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: cluster 2023-10-09T04:22:21.572709+0000 mon.a (mon.0) 311 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:22:22.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:22 smithi082 bash[79733]: audit 2023-10-09T04:22:21.573625+0000 mon.a (mon.0) 312 : audit 1 from='osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:23.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: cluster 2023-10-09T04:22:22.569450+0000 mon.a (mon.0) 313 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:24.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: cluster 2023-10-09T04:22:22.572886+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:24.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: cluster 2023-10-09T04:22:22.576370+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:22:24.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: cluster 2023-10-09T04:22:22.576432+0000 mon.a (mon.0) 314 : cluster 1 osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288] boot 2023-10-09T04:22:24.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: cluster 2023-10-09T04:22:22.576527+0000 mon.a (mon.0) 315 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:22:24.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: audit 2023-10-09T04:22:22.578420+0000 mon.a (mon.0) 316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:23 smithi153 bash[59565]: audit 2023-10-09T04:22:23.029913+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.82:0/2677572634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: cluster 2023-10-09T04:22:22.569450+0000 mon.a (mon.0) 313 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: cluster 2023-10-09T04:22:22.572886+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: cluster 2023-10-09T04:22:22.576370+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: cluster 2023-10-09T04:22:22.576432+0000 mon.a (mon.0) 314 : cluster 1 osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288] boot 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: cluster 2023-10-09T04:22:22.576527+0000 mon.a (mon.0) 315 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: audit 2023-10-09T04:22:22.578420+0000 mon.a (mon.0) 316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[77063]: audit 2023-10-09T04:22:23.029913+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.82:0/2677572634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: cluster 2023-10-09T04:22:22.569450+0000 mon.a (mon.0) 313 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: cluster 2023-10-09T04:22:22.572886+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: cluster 2023-10-09T04:22:22.576370+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: cluster 2023-10-09T04:22:22.576432+0000 mon.a (mon.0) 314 : cluster 1 osd.0 [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288] boot 2023-10-09T04:22:24.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: cluster 2023-10-09T04:22:22.576527+0000 mon.a (mon.0) 315 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T04:22:24.193 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: audit 2023-10-09T04:22:22.578420+0000 mon.a (mon.0) 316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:22:24.193 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:23 smithi082 bash[79733]: audit 2023-10-09T04:22:23.029913+0000 mon.a (mon.0) 317 : audit 0 from='client.? 172.21.15.82:0/2677572634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.236183+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v154: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:24.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.591092+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.594132+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.594834+0000 mon.a (mon.0) 318 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:22:24.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.595436+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.945 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.597774+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.597877+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.598684+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.598902+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.598947+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.599618+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.599989+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:24 smithi153 bash[59565]: cluster 2023-10-09T04:22:23.601897+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.236183+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v154: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.591092+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.594132+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.594834+0000 mon.a (mon.0) 318 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.595436+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.597774+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.597877+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.598684+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.598902+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.598947+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.599618+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.599989+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[77063]: cluster 2023-10-09T04:22:23.601897+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.236183+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v154: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T04:22:25.034 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.591092+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.594132+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.594834+0000 mon.a (mon.0) 318 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.595436+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.597774+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.597877+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.598684+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.598902+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.598947+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.599618+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.599989+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.035 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:24 smithi082 bash[79733]: cluster 2023-10-09T04:22:23.601897+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T04:22:25.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:25 smithi082 bash[77063]: audit 2023-10-09T04:22:25.384031+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.82:0/2571970734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:25.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:25 smithi082 bash[79733]: audit 2023-10-09T04:22:25.384031+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.82:0/2571970734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:25.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:25 smithi153 bash[59565]: audit 2023-10-09T04:22:25.384031+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.82:0/2571970734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:26.814 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:26 smithi082 bash[79733]: cluster 2023-10-09T04:22:25.237319+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:26.814 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:26 smithi082 bash[79733]: cluster 2023-10-09T04:22:25.601915+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:26.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:26 smithi082 bash[77063]: cluster 2023-10-09T04:22:25.237319+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:26.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:26 smithi082 bash[77063]: cluster 2023-10-09T04:22:25.601915+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:26.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:26 smithi153 bash[59565]: cluster 2023-10-09T04:22:25.237319+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v156: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:26.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:26 smithi153 bash[59565]: cluster 2023-10-09T04:22:25.601915+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:27 smithi153 bash[59565]: cluster 2023-10-09T04:22:27.613966+0000 mon.a (mon.0) 321 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T04:22:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:27 smithi153 bash[59565]: cluster 2023-10-09T04:22:27.613992+0000 mon.a (mon.0) 322 : cluster 1 Cluster is now healthy 2023-10-09T04:22:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:27 smithi082 bash[79733]: cluster 2023-10-09T04:22:27.613966+0000 mon.a (mon.0) 321 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T04:22:27.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:27 smithi082 bash[79733]: cluster 2023-10-09T04:22:27.613992+0000 mon.a (mon.0) 322 : cluster 1 Cluster is now healthy 2023-10-09T04:22:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:27 smithi082 bash[77063]: cluster 2023-10-09T04:22:27.613966+0000 mon.a (mon.0) 321 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T04:22:27.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:27 smithi082 bash[77063]: cluster 2023-10-09T04:22:27.613992+0000 mon.a (mon.0) 322 : cluster 1 Cluster is now healthy 2023-10-09T04:22:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:28 smithi153 bash[59565]: cluster 2023-10-09T04:22:27.237977+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:28.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:28 smithi153 bash[59565]: audit 2023-10-09T04:22:27.738247+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.82:0/1155929567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:28 smithi082 bash[77063]: cluster 2023-10-09T04:22:27.237977+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:28 smithi082 bash[77063]: audit 2023-10-09T04:22:27.738247+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.82:0/1155929567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:28 smithi082 bash[79733]: cluster 2023-10-09T04:22:27.237977+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:28 smithi082 bash[79733]: audit 2023-10-09T04:22:27.738247+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.82:0/1155929567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:29] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.33.4" 2023-10-09T04:22:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: cluster 2023-10-09T04:22:29.238534+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.253269+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.254512+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.814092+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.821829+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.823366+0000 mon.a (mon.0) 327 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.824497+0000 mon.a (mon.0) 328 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.830983+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.913246+0000 mon.a (mon.0) 330 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.918430+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.921817+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.923921+0000 mon.a (mon.0) 333 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:29.926080+0000 mon.a (mon.0) 334 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[77063]: audit 2023-10-09T04:22:30.108657+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.82:0/3065227968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: cluster 2023-10-09T04:22:29.238534+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.253269+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.254512+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.814092+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.821829+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.823366+0000 mon.a (mon.0) 327 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:30.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.824497+0000 mon.a (mon.0) 328 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.830983+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.913246+0000 mon.a (mon.0) 330 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.918430+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.921817+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.923921+0000 mon.a (mon.0) 333 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:29.926080+0000 mon.a (mon.0) 334 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:30.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:30 smithi082 bash[79733]: audit 2023-10-09T04:22:30.108657+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.82:0/3065227968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: cluster 2023-10-09T04:22:29.238534+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.253269+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.254512+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.814092+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.821829+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.823366+0000 mon.a (mon.0) 327 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.824497+0000 mon.a (mon.0) 328 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.830983+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.913246+0000 mon.a (mon.0) 330 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.918430+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.921817+0000 mon.a (mon.0) 332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.923921+0000 mon.a (mon.0) 333 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:29.926080+0000 mon.a (mon.0) 334 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:30.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:30 smithi153 bash[59565]: audit 2023-10-09T04:22:30.108657+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.82:0/3065227968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:31.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:31 smithi082 bash[77063]: audit 2023-10-09T04:22:29.926308+0000 mgr.y (mgr.34107) 221 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:31.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:31 smithi082 bash[77063]: cephadm 2023-10-09T04:22:29.927658+0000 mgr.y (mgr.34107) 222 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:22:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:31 smithi082 bash[79733]: audit 2023-10-09T04:22:29.926308+0000 mgr.y (mgr.34107) 221 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:31 smithi082 bash[79733]: cephadm 2023-10-09T04:22:29.927658+0000 mgr.y (mgr.34107) 222 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:22:31.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:31 smithi153 bash[59565]: audit 2023-10-09T04:22:29.926308+0000 mgr.y (mgr.34107) 221 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:31.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:31 smithi153 bash[59565]: cephadm 2023-10-09T04:22:29.927658+0000 mgr.y (mgr.34107) 222 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T04:22:32.551 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[79733]: cluster 2023-10-09T04:22:31.239571+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:32.551 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[79733]: audit 2023-10-09T04:22:32.253148+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:32.551 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[79733]: audit 2023-10-09T04:22:32.254580+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:22:32.551 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[79733]: audit 2023-10-09T04:22:32.255799+0000 mon.a (mon.0) 337 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:32.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[77063]: cluster 2023-10-09T04:22:31.239571+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:32.552 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[77063]: audit 2023-10-09T04:22:32.253148+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:32.552 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[77063]: audit 2023-10-09T04:22:32.254580+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:22:32.552 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:32 smithi082 bash[77063]: audit 2023-10-09T04:22:32.255799+0000 mon.a (mon.0) 337 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:32 smithi153 bash[59565]: cluster 2023-10-09T04:22:31.239571+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:32 smithi153 bash[59565]: audit 2023-10-09T04:22:32.253148+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:32 smithi153 bash[59565]: audit 2023-10-09T04:22:32.254580+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:22:32.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:32 smithi153 bash[59565]: audit 2023-10-09T04:22:32.255799+0000 mon.a (mon.0) 337 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:33.629 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[77063]: cephadm 2023-10-09T04:22:32.246822+0000 mgr.y (mgr.34107) 224 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:22:33.630 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[77063]: cephadm 2023-10-09T04:22:32.258584+0000 mgr.y (mgr.34107) 225 : cephadm 1 Deploying daemon osd.1 on smithi082 2023-10-09T04:22:33.630 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[77063]: audit 2023-10-09T04:22:32.476908+0000 mon.a (mon.0) 338 : audit 0 from='client.? 172.21.15.82:0/2652694167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:33.630 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[79733]: cephadm 2023-10-09T04:22:32.246822+0000 mgr.y (mgr.34107) 224 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:22:33.630 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[79733]: cephadm 2023-10-09T04:22:32.258584+0000 mgr.y (mgr.34107) 225 : cephadm 1 Deploying daemon osd.1 on smithi082 2023-10-09T04:22:33.630 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:33 smithi082 bash[79733]: audit 2023-10-09T04:22:32.476908+0000 mon.a (mon.0) 338 : audit 0 from='client.? 172.21.15.82:0/2652694167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:33.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:33 smithi153 bash[59565]: cephadm 2023-10-09T04:22:32.246822+0000 mgr.y (mgr.34107) 224 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T04:22:33.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:33 smithi153 bash[59565]: cephadm 2023-10-09T04:22:32.258584+0000 mgr.y (mgr.34107) 225 : cephadm 1 Deploying daemon osd.1 on smithi082 2023-10-09T04:22:33.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:33 smithi153 bash[59565]: audit 2023-10-09T04:22:32.476908+0000 mon.a (mon.0) 338 : audit 0 from='client.? 172.21.15.82:0/2652694167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:34 smithi153 bash[59565]: cluster 2023-10-09T04:22:33.240290+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:34 smithi082 bash[77063]: cluster 2023-10-09T04:22:33.240290+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:34.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:34 smithi082 bash[79733]: cluster 2023-10-09T04:22:33.240290+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:35 smithi153 bash[59565]: audit 2023-10-09T04:22:34.293866+0000 mon.a (mon.0) 339 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:35.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:35 smithi153 bash[59565]: audit 2023-10-09T04:22:34.825152+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.82:0/3380980162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:35.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:35 smithi082 bash[79733]: audit 2023-10-09T04:22:34.293866+0000 mon.a (mon.0) 339 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:35.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:35 smithi082 bash[79733]: audit 2023-10-09T04:22:34.825152+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.82:0/3380980162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:35.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:35 smithi082 bash[77063]: audit 2023-10-09T04:22:34.293866+0000 mon.a (mon.0) 339 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:35.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:35 smithi082 bash[77063]: audit 2023-10-09T04:22:34.825152+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.82:0/3380980162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:36.633 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:36 smithi082 bash[79733]: cluster 2023-10-09T04:22:35.241341+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:36.633 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:36 smithi082 bash[77063]: cluster 2023-10-09T04:22:35.241341+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:36.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:36 smithi153 bash[59565]: cluster 2023-10-09T04:22:35.241341+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:36.883 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:36 smithi082 systemd[1]: Stopping Ceph osd.1 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:22:37.304 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[41123]: debug 2023-10-09T04:22:37.021+0000 7f774c0e5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:22:37.305 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[41123]: debug 2023-10-09T04:22:37.021+0000 7f774c0e5700 -1 osd.1 89 *** Got signal Terminated *** 2023-10-09T04:22:37.305 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[41123]: debug 2023-10-09T04:22:37.021+0000 7f774c0e5700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:22:37.578 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.027498+0000 mon.a (mon.0) 340 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:22:37.578 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[79733]: audit 2023-10-09T04:22:37.185359+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.82:0/277075695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:37.579 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.027498+0000 mon.a (mon.0) 340 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:22:37.579 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[77063]: audit 2023-10-09T04:22:37.185359+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.82:0/277075695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:37.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:37 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.027498+0000 mon.a (mon.0) 340 : cluster 1 osd.1 marked itself down and dead 2023-10-09T04:22:37.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:37 smithi153 bash[59565]: audit 2023-10-09T04:22:37.185359+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.82:0/277075695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:38.232 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:37 smithi082 bash[87776]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T04:22:38.232 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[87867]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.242009+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.301707+0000 mon.a (mon.0) 342 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.311749+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.312149+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[77063]: cluster 2023-10-09T04:22:37.315235+0000 mon.a (mon.0) 343 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.242009+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.301707+0000 mon.a (mon.0) 342 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.311749+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.312149+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:38 smithi082 bash[79733]: cluster 2023-10-09T04:22:37.315235+0000 mon.a (mon.0) 343 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:22:38.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:38 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.242009+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:38 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.301707+0000 mon.a (mon.0) 342 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:22:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:38 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.311749+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:38 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.312149+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T04:22:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:38 smithi153 bash[59565]: cluster 2023-10-09T04:22:37.315235+0000 mon.a (mon.0) 343 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:22:39.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.315062+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.317125+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.317360+0000 mon.a (mon.0) 344 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.317648+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.318662+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.319446+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.321232+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.321639+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.322448+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.323755+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[77063]: cluster 2023-10-09T04:22:38.324204+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.315062+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.317125+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.317360+0000 mon.a (mon.0) 344 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.317648+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.318662+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.319446+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.321232+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.321639+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.322448+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.323755+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[79733]: cluster 2023-10-09T04:22:38.324204+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.315062+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.317125+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.317360+0000 mon.a (mon.0) 344 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.317648+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.318662+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.319446+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.321232+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.321639+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.322448+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.323755+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:39 smithi153 bash[59565]: cluster 2023-10-09T04:22:38.324204+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T04:22:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:39] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.33.4" 2023-10-09T04:22:40.601 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:40 smithi082 bash[77063]: cluster 2023-10-09T04:22:39.242627+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:40.602 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:40 smithi082 bash[77063]: audit 2023-10-09T04:22:39.551581+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.82:0/2158941060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.602 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:40 smithi082 bash[79733]: cluster 2023-10-09T04:22:39.242627+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:40.602 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:40 smithi082 bash[79733]: audit 2023-10-09T04:22:39.551581+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.82:0/2158941060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.602 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:40 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T04:22:40.602 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:40 smithi082 systemd[1]: Stopped Ceph osd.1 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:22:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:40 smithi153 bash[59565]: cluster 2023-10-09T04:22:39.242627+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:40.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:40 smithi153 bash[59565]: audit 2023-10-09T04:22:39.551581+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.82:0/2158941060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.982 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:40 smithi082 systemd[1]: Started Ceph osd.1 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:40.633462+0000 mon.a (mon.0) 345 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:40.641724+0000 mon.a (mon.0) 346 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:40.690095+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:41.067676+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:41.073234+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:41.077759+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:41 smithi153 bash[59565]: audit 2023-10-09T04:22:41.083006+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:40.633462+0000 mon.a (mon.0) 345 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:40.641724+0000 mon.a (mon.0) 346 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:40.690095+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:41.067676+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:41.073234+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:41.077759+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[79733]: audit 2023-10-09T04:22:41.083006+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.988 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:40.633462+0000 mon.a (mon.0) 345 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:40.641724+0000 mon.a (mon.0) 346 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:40.690095+0000 mon.a (mon.0) 347 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:41.067676+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:41.073234+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:41.077759+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:41.989 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:41 smithi082 bash[77063]: audit 2023-10-09T04:22:41.083006+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:42.482 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:22:42.482 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:22:42.482 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T04:22:42.482 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:22:42.483 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:22:42.483 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T04:22:42.483 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:22:42.483 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[88048]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T04:22:42.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[79733]: cluster 2023-10-09T04:22:41.243330+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v166: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 43 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:42.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[79733]: audit 2023-10-09T04:22:41.928370+0000 mon.a (mon.0) 352 : audit 0 from='client.? 172.21.15.82:0/2356186837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:42.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[79733]: cluster 2023-10-09T04:22:42.080232+0000 mon.a (mon.0) 353 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:42.887 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[77063]: cluster 2023-10-09T04:22:41.243330+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v166: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 43 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:42.887 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[77063]: audit 2023-10-09T04:22:41.928370+0000 mon.a (mon.0) 352 : audit 0 from='client.? 172.21.15.82:0/2356186837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:42.887 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:42 smithi082 bash[77063]: cluster 2023-10-09T04:22:42.080232+0000 mon.a (mon.0) 353 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:42.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:42 smithi153 bash[59565]: cluster 2023-10-09T04:22:41.243330+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v166: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 43 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:22:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:42 smithi153 bash[59565]: audit 2023-10-09T04:22:41.928370+0000 mon.a (mon.0) 352 : audit 0 from='client.? 172.21.15.82:0/2356186837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:42.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:42 smithi153 bash[59565]: cluster 2023-10-09T04:22:42.080232+0000 mon.a (mon.0) 353 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:43.982 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:43 smithi082 bash[88365]: debug 2023-10-09T04:22:43.841+0000 7f9ffc2fc700 1 -- 172.21.15.82:0/2750560176 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c5f8f2a4e0 con 0x55c5f8f84800 2023-10-09T04:22:44.732 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[88365]: debug 2023-10-09T04:22:44.397+0000 7fa005482640 -1 Falling back to public interface 2023-10-09T04:22:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[77063]: cluster 2023-10-09T04:22:43.244119+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v167: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[77063]: audit 2023-10-09T04:22:44.254393+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[77063]: audit 2023-10-09T04:22:44.255878+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[77063]: audit 2023-10-09T04:22:44.291564+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.82:0/2550148980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[79733]: cluster 2023-10-09T04:22:43.244119+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v167: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[79733]: audit 2023-10-09T04:22:44.254393+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[79733]: audit 2023-10-09T04:22:44.255878+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:44.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:44 smithi082 bash[79733]: audit 2023-10-09T04:22:44.291564+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.82:0/2550148980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:44 smithi153 bash[59565]: cluster 2023-10-09T04:22:43.244119+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v167: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:44.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:44 smithi153 bash[59565]: audit 2023-10-09T04:22:44.254393+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:44.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:44 smithi153 bash[59565]: audit 2023-10-09T04:22:44.255878+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:44.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:44 smithi153 bash[59565]: audit 2023-10-09T04:22:44.291564+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.82:0/2550148980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:46.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:46 smithi153 bash[59565]: cluster 2023-10-09T04:22:45.245242+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:46 smithi082 bash[77063]: cluster 2023-10-09T04:22:45.245242+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:46.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:46 smithi082 bash[79733]: cluster 2023-10-09T04:22:45.245242+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:47 smithi153 bash[59565]: audit 2023-10-09T04:22:46.669223+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.82:0/3349751377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:47 smithi153 bash[59565]: audit 2023-10-09T04:22:47.528645+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:47.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:47 smithi153 bash[59565]: audit 2023-10-09T04:22:47.536150+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:47.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[77063]: audit 2023-10-09T04:22:46.669223+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.82:0/3349751377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:47.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[77063]: audit 2023-10-09T04:22:47.528645+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:47.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[77063]: audit 2023-10-09T04:22:47.536150+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:47.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[79733]: audit 2023-10-09T04:22:46.669223+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.82:0/3349751377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:47.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[79733]: audit 2023-10-09T04:22:47.528645+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:47.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[79733]: audit 2023-10-09T04:22:47.536150+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.232 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:47 smithi082 bash[88365]: debug 2023-10-09T04:22:47.953+0000 7fa005482640 -1 osd.1 89 log_to_monitors true 2023-10-09T04:22:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:48 smithi153 bash[59565]: cluster 2023-10-09T04:22:47.245942+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:48 smithi153 bash[59565]: audit 2023-10-09T04:22:47.963249+0000 mon.a (mon.0) 360 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:22:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:48 smithi153 bash[59565]: audit 2023-10-09T04:22:48.329058+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:48 smithi153 bash[59565]: audit 2023-10-09T04:22:48.337791+0000 mon.a (mon.0) 362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.982 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[88365]: debug 2023-10-09T04:22:48.705+0000 7f9ff48ea700 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:22:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[77063]: cluster 2023-10-09T04:22:47.245942+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[77063]: audit 2023-10-09T04:22:47.963249+0000 mon.a (mon.0) 360 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:22:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[77063]: audit 2023-10-09T04:22:48.329058+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[77063]: audit 2023-10-09T04:22:48.337791+0000 mon.a (mon.0) 362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[79733]: cluster 2023-10-09T04:22:47.245942+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[79733]: audit 2023-10-09T04:22:47.963249+0000 mon.a (mon.0) 360 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:22:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[79733]: audit 2023-10-09T04:22:48.329058+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:48.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:48 smithi082 bash[79733]: audit 2023-10-09T04:22:48.337791+0000 mon.a (mon.0) 362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:49] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.33.4" 2023-10-09T04:22:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:48.659004+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: audit 2023-10-09T04:22:48.663610+0000 mon.a (mon.0) 363 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:48.663705+0000 mon.a (mon.0) 364 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: audit 2023-10-09T04:22:48.664621+0000 mon.a (mon.0) 365 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:48.669177+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: audit 2023-10-09T04:22:49.026060+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.82:0/3728473603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.031038+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.039196+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.218390+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:48.659004+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: audit 2023-10-09T04:22:48.663610+0000 mon.a (mon.0) 363 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:48.663705+0000 mon.a (mon.0) 364 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: audit 2023-10-09T04:22:48.664621+0000 mon.a (mon.0) 365 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:48.669177+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: audit 2023-10-09T04:22:49.026060+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.82:0/3728473603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.031038+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.039196+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:49.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:49 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.218390+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:48.659004+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: audit 2023-10-09T04:22:48.663610+0000 mon.a (mon.0) 363 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:48.663705+0000 mon.a (mon.0) 364 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: audit 2023-10-09T04:22:48.664621+0000 mon.a (mon.0) 365 : audit 1 from='osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:48.669177+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: audit 2023-10-09T04:22:49.026060+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.82:0/3728473603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.031038+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.039196+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T04:22:50.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:49 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.218390+0000 mon.a (mon.0) 366 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.246657+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.668147+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.673051+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.676177+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.676972+0000 mon.a (mon.0) 368 : cluster 1 osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898] boot 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:49.677088+0000 mon.a (mon.0) 369 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: audit 2023-10-09T04:22:49.695874+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.680114+0000 mon.a (mon.0) 371 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.246657+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.668147+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.673051+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.676177+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:50.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.676972+0000 mon.a (mon.0) 368 : cluster 1 osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898] boot 2023-10-09T04:22:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:49.677088+0000 mon.a (mon.0) 369 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:22:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: audit 2023-10-09T04:22:49.695874+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:22:50.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:50 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.680114+0000 mon.a (mon.0) 371 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.246657+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.668147+0000 mon.a (mon.0) 367 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.673051+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.676177+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e93 with expected crc 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.676972+0000 mon.a (mon.0) 368 : cluster 1 osd.1 [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898] boot 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:49.677088+0000 mon.a (mon.0) 369 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: audit 2023-10-09T04:22:49.695874+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:22:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:50 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.680114+0000 mon.a (mon.0) 371 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:22:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.676405+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.678746+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.678952+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.679867+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.680383+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.681582+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.682449+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.682811+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.684054+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.684991+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: cluster 2023-10-09T04:22:50.685024+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[77063]: audit 2023-10-09T04:22:51.391744+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.82:0/866204201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:51.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.676405+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.678746+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.678952+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.679867+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.680383+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.681582+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.682449+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.682811+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.684054+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.684991+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: cluster 2023-10-09T04:22:50.685024+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:51.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:51 smithi082 bash[79733]: audit 2023-10-09T04:22:51.391744+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.82:0/866204201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.676405+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.678746+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.678952+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.679867+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.680383+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.681582+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.682449+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.682811+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.684054+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.684991+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: cluster 2023-10-09T04:22:50.685024+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T04:22:52.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:51 smithi153 bash[59565]: audit 2023-10-09T04:22:51.391744+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.82:0/866204201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:52.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:52 smithi082 bash[77063]: cluster 2023-10-09T04:22:51.247327+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v174: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:22:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:52 smithi082 bash[79733]: cluster 2023-10-09T04:22:51.247327+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v174: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:22:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:52 smithi153 bash[59565]: cluster 2023-10-09T04:22:51.247327+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v174: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:22:54.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[77063]: cluster 2023-10-09T04:22:53.248214+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v175: 73 pgs: 20 peering, 1 active+undersized+degraded, 9 active+undersized, 43 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T04:22:54.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[77063]: audit 2023-10-09T04:22:53.760005+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.82:0/897885870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:54.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[77063]: cluster 2023-10-09T04:22:54.220186+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:22:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[79733]: cluster 2023-10-09T04:22:53.248214+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v175: 73 pgs: 20 peering, 1 active+undersized+degraded, 9 active+undersized, 43 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T04:22:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[79733]: audit 2023-10-09T04:22:53.760005+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.82:0/897885870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:54 smithi082 bash[79733]: cluster 2023-10-09T04:22:54.220186+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:22:55.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:54 smithi153 bash[59565]: cluster 2023-10-09T04:22:53.248214+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v175: 73 pgs: 20 peering, 1 active+undersized+degraded, 9 active+undersized, 43 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T04:22:55.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:54 smithi153 bash[59565]: audit 2023-10-09T04:22:53.760005+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.82:0/897885870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:55.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:54 smithi153 bash[59565]: cluster 2023-10-09T04:22:54.220186+0000 mon.a (mon.0) 372 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:22:56.022 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[77063]: cluster 2023-10-09T04:22:55.249267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:56.023 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[77063]: cluster 2023-10-09T04:22:55.709166+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T04:22:56.023 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[77063]: cluster 2023-10-09T04:22:55.709210+0000 mon.a (mon.0) 374 : cluster 1 Cluster is now healthy 2023-10-09T04:22:56.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[79733]: cluster 2023-10-09T04:22:55.249267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:56.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[79733]: cluster 2023-10-09T04:22:55.709166+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T04:22:56.023 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:55 smithi082 bash[79733]: cluster 2023-10-09T04:22:55.709210+0000 mon.a (mon.0) 374 : cluster 1 Cluster is now healthy 2023-10-09T04:22:56.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:55 smithi153 bash[59565]: cluster 2023-10-09T04:22:55.249267+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v176: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:56.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:55 smithi153 bash[59565]: cluster 2023-10-09T04:22:55.709166+0000 mon.a (mon.0) 373 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T04:22:56.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:55 smithi153 bash[59565]: cluster 2023-10-09T04:22:55.709210+0000 mon.a (mon.0) 374 : cluster 1 Cluster is now healthy 2023-10-09T04:22:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:56 smithi153 bash[59565]: audit 2023-10-09T04:22:56.124560+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.82:0/1853891377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:57.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:56 smithi082 bash[77063]: audit 2023-10-09T04:22:56.124560+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.82:0/1853891377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:57.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:56 smithi082 bash[79733]: audit 2023-10-09T04:22:56.124560+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.82:0/1853891377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: cluster 2023-10-09T04:22:57.249745+0000 mgr.y (mgr.34107) 238 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.483016+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.491161+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.492774+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.494351+0000 mon.a (mon.0) 378 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.501491+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.585615+0000 mon.a (mon.0) 380 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.592003+0000 mon.a (mon.0) 381 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.595152+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.596606+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.598515+0000 mon.a (mon.0) 384 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:57.598752+0000 mgr.y (mgr.34107) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: cephadm 2023-10-09T04:22:57.599930+0000 mgr.y (mgr.34107) 240 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[79733]: audit 2023-10-09T04:22:58.472108+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.82:0/697985987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: cluster 2023-10-09T04:22:57.249745+0000 mgr.y (mgr.34107) 238 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:58.801 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.483016+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.491161+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.492774+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.494351+0000 mon.a (mon.0) 378 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.501491+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.585615+0000 mon.a (mon.0) 380 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.592003+0000 mon.a (mon.0) 381 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.595152+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.596606+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.802 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.598515+0000 mon.a (mon.0) 384 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.803 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:57.598752+0000 mgr.y (mgr.34107) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.803 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: cephadm 2023-10-09T04:22:57.599930+0000 mgr.y (mgr.34107) 240 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T04:22:58.803 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:22:58 smithi082 bash[77063]: audit 2023-10-09T04:22:58.472108+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.82:0/697985987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:58.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: cluster 2023-10-09T04:22:57.249745+0000 mgr.y (mgr.34107) 238 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:22:58.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.483016+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.491161+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.492774+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.494351+0000 mon.a (mon.0) 378 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.501491+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.585615+0000 mon.a (mon.0) 380 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.592003+0000 mon.a (mon.0) 381 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.595152+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.596606+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.598515+0000 mon.a (mon.0) 384 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:57.598752+0000 mgr.y (mgr.34107) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: cephadm 2023-10-09T04:22:57.599930+0000 mgr.y (mgr.34107) 240 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T04:22:58.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:22:58 smithi153 bash[59565]: audit 2023-10-09T04:22:58.472108+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.82:0/697985987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:22:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:22:59] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-09T04:23:00.389 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: cluster 2023-10-09T04:22:59.250181+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:00.389 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: audit 2023-10-09T04:22:59.254042+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.389 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: audit 2023-10-09T04:22:59.255266+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:00.389 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: audit 2023-10-09T04:22:59.779473+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.390 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: audit 2023-10-09T04:22:59.791949+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:23:00.390 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[77063]: audit 2023-10-09T04:22:59.793367+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: cluster 2023-10-09T04:22:59.250181+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: audit 2023-10-09T04:22:59.254042+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: audit 2023-10-09T04:22:59.255266+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: audit 2023-10-09T04:22:59.779473+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: audit 2023-10-09T04:22:59.791949+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:23:00.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:00 smithi082 bash[79733]: audit 2023-10-09T04:22:59.793367+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: cluster 2023-10-09T04:22:59.250181+0000 mgr.y (mgr.34107) 241 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: audit 2023-10-09T04:22:59.254042+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: audit 2023-10-09T04:22:59.255266+0000 mon.a (mon.0) 386 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: audit 2023-10-09T04:22:59.779473+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: audit 2023-10-09T04:22:59.791949+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:23:00.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:00 smithi153 bash[59565]: audit 2023-10-09T04:22:59.793367+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[77063]: cephadm 2023-10-09T04:22:59.769969+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T04:23:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[77063]: cephadm 2023-10-09T04:22:59.797228+0000 mgr.y (mgr.34107) 243 : cephadm 1 Deploying daemon osd.2 on smithi082 2023-10-09T04:23:01.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[77063]: audit 2023-10-09T04:23:00.826930+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.82:0/3914625148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[79733]: cephadm 2023-10-09T04:22:59.769969+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T04:23:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[79733]: cephadm 2023-10-09T04:22:59.797228+0000 mgr.y (mgr.34107) 243 : cephadm 1 Deploying daemon osd.2 on smithi082 2023-10-09T04:23:01.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:01 smithi082 bash[79733]: audit 2023-10-09T04:23:00.826930+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.82:0/3914625148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:01.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:01 smithi153 bash[59565]: cephadm 2023-10-09T04:22:59.769969+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T04:23:01.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:01 smithi153 bash[59565]: cephadm 2023-10-09T04:22:59.797228+0000 mgr.y (mgr.34107) 243 : cephadm 1 Deploying daemon osd.2 on smithi082 2023-10-09T04:23:01.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:01 smithi153 bash[59565]: audit 2023-10-09T04:23:00.826930+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.82:0/3914625148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:02 smithi153 bash[59565]: cluster 2023-10-09T04:23:01.251342+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:02.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:02 smithi082 bash[77063]: cluster 2023-10-09T04:23:01.251342+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:02.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:02 smithi082 bash[79733]: cluster 2023-10-09T04:23:01.251342+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:03.426 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:03 smithi082 bash[79733]: audit 2023-10-09T04:23:03.188273+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.82:0/1916283263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:03.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:03 smithi082 bash[77063]: audit 2023-10-09T04:23:03.188273+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.82:0/1916283263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:03.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:03 smithi153 bash[59565]: audit 2023-10-09T04:23:03.188273+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.82:0/1916283263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:04.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:04 smithi153 bash[59565]: cluster 2023-10-09T04:23:03.252044+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:04.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:04 smithi082 bash[77063]: cluster 2023-10-09T04:23:03.252044+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:04.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:04 smithi082 bash[79733]: cluster 2023-10-09T04:23:03.252044+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:04.701 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:04 smithi082 systemd[1]: Stopping Ceph osd.2 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:23:04.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:04 smithi082 bash[45275]: debug 2023-10-09T04:23:04.693+0000 7fd89d91d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:23:04.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:04 smithi082 bash[45275]: debug 2023-10-09T04:23:04.693+0000 7fd89d91d700 -1 osd.2 94 *** Got signal Terminated *** 2023-10-09T04:23:04.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:04 smithi082 bash[45275]: debug 2023-10-09T04:23:04.693+0000 7fd89d91d700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:23:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:05 smithi153 bash[59565]: audit 2023-10-09T04:23:04.314355+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:05.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:05 smithi153 bash[59565]: cluster 2023-10-09T04:23:04.698941+0000 mon.a (mon.0) 391 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:23:05.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:05 smithi082 bash[77063]: audit 2023-10-09T04:23:04.314355+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:05.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:05 smithi082 bash[77063]: cluster 2023-10-09T04:23:04.698941+0000 mon.a (mon.0) 391 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:23:05.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:05 smithi082 bash[79733]: audit 2023-10-09T04:23:04.314355+0000 mon.a (mon.0) 390 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:05.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:05 smithi082 bash[79733]: cluster 2023-10-09T04:23:04.698941+0000 mon.a (mon.0) 391 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:23:06.205 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:05 smithi082 bash[90562]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T04:23:06.205 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[90652]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.253175+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.310580+0000 mon.a (mon.0) 392 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.318628+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.320268+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.320853+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.321200+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.322588+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: cluster 2023-10-09T04:23:05.322875+0000 mon.a (mon.0) 393 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[77063]: audit 2023-10-09T04:23:05.560733+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.82:0/1031839795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.253175+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.310580+0000 mon.a (mon.0) 392 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.318628+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.320268+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.320853+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.321200+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.322588+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: cluster 2023-10-09T04:23:05.322875+0000 mon.a (mon.0) 393 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:23:06.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:06 smithi082 bash[79733]: audit 2023-10-09T04:23:05.560733+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.82:0/1031839795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.253175+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.310580+0000 mon.a (mon.0) 392 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.318628+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.320268+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.320853+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.321200+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.322588+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: cluster 2023-10-09T04:23:05.322875+0000 mon.a (mon.0) 393 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T04:23:06.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:06 smithi153 bash[59565]: audit 2023-10-09T04:23:05.560733+0000 mon.a (mon.0) 394 : audit 0 from='client.? 172.21.15.82:0/1031839795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.325503+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.326222+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.326827+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.326880+0000 mon.a (mon.0) 395 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.328459+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.328831+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.329093+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.329191+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.329310+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[79733]: cluster 2023-10-09T04:23:06.329891+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.325503+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.326222+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.326827+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.326880+0000 mon.a (mon.0) 395 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.328459+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.328831+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.329093+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.329191+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.329310+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:07 smithi082 bash[77063]: cluster 2023-10-09T04:23:06.329891+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.325503+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.326222+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.326827+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.326880+0000 mon.a (mon.0) 395 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.328459+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.328831+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.329093+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.329191+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.329310+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:07.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:07 smithi153 bash[59565]: cluster 2023-10-09T04:23:06.329891+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T04:23:08.589 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:08 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T04:23:08.589 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:08 smithi082 systemd[1]: Stopped Ceph osd.2 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[77063]: cluster 2023-10-09T04:23:07.253931+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v184: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 2 active+undersized, 68 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[77063]: cluster 2023-10-09T04:23:07.333944+0000 mon.a (mon.0) 396 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[77063]: audit 2023-10-09T04:23:07.924029+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.82:0/4014899448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[79733]: cluster 2023-10-09T04:23:07.253931+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v184: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 2 active+undersized, 68 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[79733]: cluster 2023-10-09T04:23:07.333944+0000 mon.a (mon.0) 396 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:08.590 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:08 smithi082 bash[79733]: audit 2023-10-09T04:23:07.924029+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.82:0/4014899448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:08.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:08 smithi153 bash[59565]: cluster 2023-10-09T04:23:07.253931+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v184: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 2 active+undersized, 68 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:08 smithi153 bash[59565]: cluster 2023-10-09T04:23:07.333944+0000 mon.a (mon.0) 396 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:08.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:08 smithi153 bash[59565]: audit 2023-10-09T04:23:07.924029+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.82:0/4014899448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:08.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:08 smithi082 systemd[1]: Started Ceph osd.2 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:09 smithi153 bash[59565]: audit 2023-10-09T04:23:08.627208+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:09 smithi153 bash[59565]: audit 2023-10-09T04:23:08.635386+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:09 smithi153 bash[59565]: audit 2023-10-09T04:23:08.680281+0000 mon.a (mon.0) 399 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[77063]: audit 2023-10-09T04:23:08.627208+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[77063]: audit 2023-10-09T04:23:08.635386+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[77063]: audit 2023-10-09T04:23:08.680281+0000 mon.a (mon.0) 399 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[79733]: audit 2023-10-09T04:23:08.627208+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[79733]: audit 2023-10-09T04:23:08.635386+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[79733]: audit 2023-10-09T04:23:08.680281+0000 mon.a (mon.0) 399 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:09.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:09] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:23:10.379 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:23:10.380 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[90832]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T04:23:10.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:10 smithi153 bash[59565]: cluster 2023-10-09T04:23:09.254520+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v185: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:10.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:10 smithi153 bash[59565]: audit 2023-10-09T04:23:10.280356+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.82:0/2881529797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[79733]: cluster 2023-10-09T04:23:09.254520+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v185: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[79733]: audit 2023-10-09T04:23:10.280356+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.82:0/2881529797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:10.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[77063]: cluster 2023-10-09T04:23:09.254520+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v185: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:10.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:10 smithi082 bash[77063]: audit 2023-10-09T04:23:10.280356+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.82:0/2881529797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:12.233 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:11 smithi082 bash[91125]: debug 2023-10-09T04:23:11.929+0000 7fb39930b700 1 -- 172.21.15.82:0/2726132535 <== mon.0 v2:172.21.15.82:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5611e84344e0 con 0x5611e848f000 2023-10-09T04:23:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:12 smithi153 bash[59565]: cluster 2023-10-09T04:23:11.255603+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v186: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:12 smithi082 bash[77063]: cluster 2023-10-09T04:23:11.255603+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v186: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:12 smithi082 bash[79733]: cluster 2023-10-09T04:23:11.255603+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v186: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:12.983 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:12 smithi082 bash[91125]: debug 2023-10-09T04:23:12.497+0000 7fb3a2491640 -1 Falling back to public interface 2023-10-09T04:23:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:13 smithi153 bash[59565]: audit 2023-10-09T04:23:12.642470+0000 mon.a (mon.0) 400 : audit 0 from='client.? 172.21.15.82:0/1485435703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:13 smithi082 bash[77063]: audit 2023-10-09T04:23:12.642470+0000 mon.a (mon.0) 400 : audit 0 from='client.? 172.21.15.82:0/1485435703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:13 smithi082 bash[79733]: audit 2023-10-09T04:23:12.642470+0000 mon.a (mon.0) 400 : audit 0 from='client.? 172.21.15.82:0/1485435703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:14 smithi153 bash[59565]: cluster 2023-10-09T04:23:13.256566+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v187: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:14 smithi153 bash[59565]: cluster 2023-10-09T04:23:14.228782+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:14.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:14 smithi153 bash[59565]: audit 2023-10-09T04:23:14.257091+0000 mon.a (mon.0) 402 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:14.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:14 smithi153 bash[59565]: audit 2023-10-09T04:23:14.258238+0000 mon.a (mon.0) 403 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[77063]: cluster 2023-10-09T04:23:13.256566+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v187: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[77063]: cluster 2023-10-09T04:23:14.228782+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[77063]: audit 2023-10-09T04:23:14.257091+0000 mon.a (mon.0) 402 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[77063]: audit 2023-10-09T04:23:14.258238+0000 mon.a (mon.0) 403 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[79733]: cluster 2023-10-09T04:23:13.256566+0000 mgr.y (mgr.34107) 250 : cluster 0 pgmap v187: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[79733]: cluster 2023-10-09T04:23:14.228782+0000 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[79733]: audit 2023-10-09T04:23:14.257091+0000 mon.a (mon.0) 402 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:14 smithi082 bash[79733]: audit 2023-10-09T04:23:14.258238+0000 mon.a (mon.0) 403 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:15.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[91125]: debug 2023-10-09T04:23:15.777+0000 7fb3a2491640 -1 osd.2 94 log_to_monitors true 2023-10-09T04:23:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:14.724882+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:14.732374+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:14.875017+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:14.882794+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:15.032716+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.82:0/1242927606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: cluster 2023-10-09T04:23:15.257683+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:15.659484+0000 mon.a (mon.0) 409 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[77063]: audit 2023-10-09T04:23:15.668121+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:14.724882+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:14.732374+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:14.875017+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:14.882794+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:15.032716+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.82:0/1242927606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: cluster 2023-10-09T04:23:15.257683+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:15.659484+0000 mon.a (mon.0) 409 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:15.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:15 smithi082 bash[79733]: audit 2023-10-09T04:23:15.668121+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:14.724882+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:14.732374+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:14.875017+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:14.882794+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:15.032716+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.82:0/1242927606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: cluster 2023-10-09T04:23:15.257683+0000 mgr.y (mgr.34107) 251 : cluster 0 pgmap v188: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:15.659484+0000 mon.a (mon.0) 409 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:15 smithi153 bash[59565]: audit 2023-10-09T04:23:15.668121+0000 mon.a (mon.0) 410 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:16.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:16 smithi082 bash[77063]: audit 2023-10-09T04:23:15.788097+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:16.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:16 smithi082 bash[77063]: audit 2023-10-09T04:23:15.788510+0000 mon.a (mon.0) 411 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:16.982 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:23:16 smithi082 bash[91125]: debug 2023-10-09T04:23:16.785+0000 7fb3910f8700 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:23:16.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:16 smithi082 bash[79733]: audit 2023-10-09T04:23:15.788097+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:16.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:16 smithi082 bash[79733]: audit 2023-10-09T04:23:15.788510+0000 mon.a (mon.0) 411 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:17.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:16 smithi153 bash[59565]: audit 2023-10-09T04:23:15.788097+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:17.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:16 smithi153 bash[59565]: audit 2023-10-09T04:23:15.788510+0000 mon.a (mon.0) 411 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:23:18.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: cluster 2023-10-09T04:23:16.732927+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: cluster 2023-10-09T04:23:16.733655+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: audit 2023-10-09T04:23:16.736259+0000 mon.a (mon.0) 412 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: cluster 2023-10-09T04:23:16.736421+0000 mon.a (mon.0) 413 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: audit 2023-10-09T04:23:16.737235+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: audit 2023-10-09T04:23:16.737778+0000 mon.a (mon.0) 414 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.258360+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:18.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:17 smithi153 bash[59565]: audit 2023-10-09T04:23:17.401346+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.82:0/278861034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: cluster 2023-10-09T04:23:16.732927+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: cluster 2023-10-09T04:23:16.733655+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: audit 2023-10-09T04:23:16.736259+0000 mon.a (mon.0) 412 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: cluster 2023-10-09T04:23:16.736421+0000 mon.a (mon.0) 413 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: audit 2023-10-09T04:23:16.737235+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: audit 2023-10-09T04:23:16.737778+0000 mon.a (mon.0) 414 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.258360+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[77063]: audit 2023-10-09T04:23:17.401346+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.82:0/278861034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: cluster 2023-10-09T04:23:16.732927+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: cluster 2023-10-09T04:23:16.733655+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: audit 2023-10-09T04:23:16.736259+0000 mon.a (mon.0) 412 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: cluster 2023-10-09T04:23:16.736421+0000 mon.a (mon.0) 413 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:23:18.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: audit 2023-10-09T04:23:16.737235+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: audit 2023-10-09T04:23:16.737778+0000 mon.a (mon.0) 414 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:18.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.258360+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v190: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:18.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:17 smithi082 bash[79733]: audit 2023-10-09T04:23:17.401346+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.82:0/278861034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:19.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.732881+0000 mon.a (mon.0) 415 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:23:19.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.745918+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.747739+0000 mon.a (mon.0) 416 : cluster 1 osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682] boot 2023-10-09T04:23:19.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.747817+0000 mon.a (mon.0) 417 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:23:19.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: audit 2023-10-09T04:23:17.748446+0000 mon.a (mon.0) 418 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:23:19.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.748886+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.750217+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.750246+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.750508+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:18 smithi153 bash[59565]: cluster 2023-10-09T04:23:17.753199+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.732881+0000 mon.a (mon.0) 415 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.745918+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.747739+0000 mon.a (mon.0) 416 : cluster 1 osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682] boot 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.747817+0000 mon.a (mon.0) 417 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: audit 2023-10-09T04:23:17.748446+0000 mon.a (mon.0) 418 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.748886+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.750217+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.750246+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.750508+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[77063]: cluster 2023-10-09T04:23:17.753199+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.732881+0000 mon.a (mon.0) 415 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.745918+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.747739+0000 mon.a (mon.0) 416 : cluster 1 osd.2 [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682] boot 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.747817+0000 mon.a (mon.0) 417 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: audit 2023-10-09T04:23:17.748446+0000 mon.a (mon.0) 418 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:23:19.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.748886+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.750217+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.750246+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.750508+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:18 smithi082 bash[79733]: cluster 2023-10-09T04:23:17.753199+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T04:23:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:19] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.33.4" 2023-10-09T04:23:19.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.752409+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.753448+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.755347+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.755595+0000 mon.a (mon.0) 419 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.755801+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.755802+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.755821+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:18.756216+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: cluster 2023-10-09T04:23:19.259050+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v193: 73 pgs: 4 peering, 3 active+undersized+degraded, 12 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[79733]: audit 2023-10-09T04:23:19.332879+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:19.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.752409+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.753448+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.755347+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.755595+0000 mon.a (mon.0) 419 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.755801+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.755802+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.755821+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:18.756216+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: cluster 2023-10-09T04:23:19.259050+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v193: 73 pgs: 4 peering, 3 active+undersized+degraded, 12 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:19.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:19 smithi082 bash[77063]: audit 2023-10-09T04:23:19.332879+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.752409+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.753448+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.755347+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.755595+0000 mon.a (mon.0) 419 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.755801+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.755802+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.755821+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:18.756216+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T04:23:20.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: cluster 2023-10-09T04:23:19.259050+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v193: 73 pgs: 4 peering, 3 active+undersized+degraded, 12 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:23:20.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:19 smithi153 bash[59565]: audit 2023-10-09T04:23:19.332879+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:20 smithi082 bash[77063]: audit 2023-10-09T04:23:19.773285+0000 mon.a (mon.0) 421 : audit 0 from='client.? 172.21.15.82:0/3064017009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:20 smithi082 bash[79733]: audit 2023-10-09T04:23:19.773285+0000 mon.a (mon.0) 421 : audit 0 from='client.? 172.21.15.82:0/3064017009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:20 smithi153 bash[59565]: audit 2023-10-09T04:23:19.773285+0000 mon.a (mon.0) 421 : audit 0 from='client.? 172.21.15.82:0/3064017009' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:21.968 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:21 smithi082 bash[79733]: cluster 2023-10-09T04:23:21.259898+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v194: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:21.968 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:21 smithi082 bash[77063]: cluster 2023-10-09T04:23:21.259898+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v194: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:21 smithi153 bash[59565]: cluster 2023-10-09T04:23:21.259898+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v194: 73 pgs: 4 peering, 1 active+undersized+degraded, 4 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:23:23.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:22 smithi082 bash[77063]: cluster 2023-10-09T04:23:21.755684+0000 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:23.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:22 smithi082 bash[77063]: audit 2023-10-09T04:23:22.141184+0000 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.82:0/2105806661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:23.146 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:22 smithi082 bash[79733]: cluster 2023-10-09T04:23:21.755684+0000 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:23.146 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:22 smithi082 bash[79733]: audit 2023-10-09T04:23:22.141184+0000 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.82:0/2105806661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:22 smithi153 bash[59565]: cluster 2023-10-09T04:23:21.755684+0000 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:23:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:22 smithi153 bash[59565]: audit 2023-10-09T04:23:22.141184+0000 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.82:0/2105806661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:24.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:23 smithi082 bash[79733]: cluster 2023-10-09T04:23:23.260632+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v195: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:24.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:23 smithi082 bash[77063]: cluster 2023-10-09T04:23:23.260632+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v195: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:24.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:23 smithi153 bash[59565]: cluster 2023-10-09T04:23:23.260632+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v195: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:24 smithi153 bash[59565]: cluster 2023-10-09T04:23:23.768804+0000 mon.a (mon.0) 424 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-09T04:23:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:24 smithi153 bash[59565]: cluster 2023-10-09T04:23:23.768841+0000 mon.a (mon.0) 425 : cluster 1 Cluster is now healthy 2023-10-09T04:23:25.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:24 smithi153 bash[59565]: audit 2023-10-09T04:23:24.495742+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.82:0/1960009188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:25.232 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[73210]: debug 2023-10-09T04:23:24.933+0000 7fed11da6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[77063]: cluster 2023-10-09T04:23:23.768804+0000 mon.a (mon.0) 424 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-09T04:23:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[77063]: cluster 2023-10-09T04:23:23.768841+0000 mon.a (mon.0) 425 : cluster 1 Cluster is now healthy 2023-10-09T04:23:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[77063]: audit 2023-10-09T04:23:24.495742+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.82:0/1960009188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[79733]: cluster 2023-10-09T04:23:23.768804+0000 mon.a (mon.0) 424 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-09T04:23:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[79733]: cluster 2023-10-09T04:23:23.768841+0000 mon.a (mon.0) 425 : cluster 1 Cluster is now healthy 2023-10-09T04:23:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:24 smithi082 bash[79733]: audit 2023-10-09T04:23:24.495742+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.82:0/1960009188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.825008+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.835434+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.837307+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.838807+0000 mon.a (mon.0) 429 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.845834+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.929191+0000 mon.a (mon.0) 431 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.933465+0000 mon.a (mon.0) 432 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.936051+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.937815+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.939935+0000 mon.a (mon.0) 435 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: audit 2023-10-09T04:23:24.940204+0000 mgr.y (mgr.34107) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: cephadm 2023-10-09T04:23:24.941414+0000 mgr.y (mgr.34107) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:26.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:25 smithi153 bash[59565]: cluster 2023-10-09T04:23:25.261675+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.825008+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.835434+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.837307+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.838807+0000 mon.a (mon.0) 429 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:23:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.845834+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.929191+0000 mon.a (mon.0) 431 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.933465+0000 mon.a (mon.0) 432 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.936051+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.937815+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.939935+0000 mon.a (mon.0) 435 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: audit 2023-10-09T04:23:24.940204+0000 mgr.y (mgr.34107) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: cephadm 2023-10-09T04:23:24.941414+0000 mgr.y (mgr.34107) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[77063]: cluster 2023-10-09T04:23:25.261675+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:26.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.825008+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.835434+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.837307+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.838807+0000 mon.a (mon.0) 429 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.845834+0000 mon.a (mon.0) 430 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.929191+0000 mon.a (mon.0) 431 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.933465+0000 mon.a (mon.0) 432 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.936051+0000 mon.a (mon.0) 433 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.937815+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.939935+0000 mon.a (mon.0) 435 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: audit 2023-10-09T04:23:24.940204+0000 mgr.y (mgr.34107) 256 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:26.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: cephadm 2023-10-09T04:23:24.941414+0000 mgr.y (mgr.34107) 257 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:26.235 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:25 smithi082 bash[79733]: cluster 2023-10-09T04:23:25.261675+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:27.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:26 smithi153 bash[59565]: audit 2023-10-09T04:23:26.858256+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.82:0/832067018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:27.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:26 smithi082 bash[77063]: audit 2023-10-09T04:23:26.858256+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.82:0/832067018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:27.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:26 smithi082 bash[79733]: audit 2023-10-09T04:23:26.858256+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.82:0/832067018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:27 smithi153 bash[59565]: cluster 2023-10-09T04:23:27.262519+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:28.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:27 smithi082 bash[77063]: cluster 2023-10-09T04:23:27.262519+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:27 smithi082 bash[79733]: cluster 2023-10-09T04:23:27.262519+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:29 smithi153 bash[59565]: audit 2023-10-09T04:23:29.225616+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.82:0/3545034402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:29 smithi153 bash[59565]: audit 2023-10-09T04:23:29.256472+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:29.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:29 smithi153 bash[59565]: audit 2023-10-09T04:23:29.257775+0000 mon.a (mon.0) 438 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[77063]: audit 2023-10-09T04:23:29.225616+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.82:0/3545034402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:29.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[77063]: audit 2023-10-09T04:23:29.256472+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:29.703 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[77063]: audit 2023-10-09T04:23:29.257775+0000 mon.a (mon.0) 438 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[79733]: audit 2023-10-09T04:23:29.225616+0000 mon.a (mon.0) 436 : audit 0 from='client.? 172.21.15.82:0/3545034402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:29.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[79733]: audit 2023-10-09T04:23:29.256472+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:29.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[79733]: audit 2023-10-09T04:23:29.257775+0000 mon.a (mon.0) 438 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:29] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.33.4" 2023-10-09T04:23:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:30 smithi153 bash[59565]: cluster 2023-10-09T04:23:29.264094+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:30 smithi153 bash[59565]: audit 2023-10-09T04:23:29.348453+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:30.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:30 smithi082 bash[77063]: cluster 2023-10-09T04:23:29.264094+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:30.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:30 smithi082 bash[77063]: audit 2023-10-09T04:23:29.348453+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:30 smithi082 bash[79733]: cluster 2023-10-09T04:23:29.264094+0000 mgr.y (mgr.34107) 260 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:30 smithi082 bash[79733]: audit 2023-10-09T04:23:29.348453+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:32 smithi153 bash[59565]: cluster 2023-10-09T04:23:31.264837+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:32 smithi153 bash[59565]: audit 2023-10-09T04:23:31.589725+0000 mon.a (mon.0) 440 : audit 0 from='client.? 172.21.15.82:0/3969069470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:32 smithi082 bash[79733]: cluster 2023-10-09T04:23:31.264837+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:32 smithi082 bash[79733]: audit 2023-10-09T04:23:31.589725+0000 mon.a (mon.0) 440 : audit 0 from='client.? 172.21.15.82:0/3969069470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:32.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:32 smithi082 bash[77063]: cluster 2023-10-09T04:23:31.264837+0000 mgr.y (mgr.34107) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:32.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:32 smithi082 bash[77063]: audit 2023-10-09T04:23:31.589725+0000 mon.a (mon.0) 440 : audit 0 from='client.? 172.21.15.82:0/3969069470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:34 smithi153 bash[59565]: cluster 2023-10-09T04:23:33.265612+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:34 smithi153 bash[59565]: audit 2023-10-09T04:23:33.951376+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.82:0/3313845971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:34 smithi082 bash[77063]: cluster 2023-10-09T04:23:33.265612+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:34 smithi082 bash[77063]: audit 2023-10-09T04:23:33.951376+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.82:0/3313845971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:34.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:34 smithi082 bash[79733]: cluster 2023-10-09T04:23:33.265612+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:34.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:34 smithi082 bash[79733]: audit 2023-10-09T04:23:33.951376+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.82:0/3313845971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:36 smithi153 bash[59565]: cluster 2023-10-09T04:23:35.266622+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:36 smithi153 bash[59565]: audit 2023-10-09T04:23:36.315709+0000 mon.a (mon.0) 442 : audit 0 from='client.? 172.21.15.82:0/1788756049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:36 smithi082 bash[77063]: cluster 2023-10-09T04:23:35.266622+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:36.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:36 smithi082 bash[77063]: audit 2023-10-09T04:23:36.315709+0000 mon.a (mon.0) 442 : audit 0 from='client.? 172.21.15.82:0/1788756049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:36.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:36 smithi082 bash[79733]: cluster 2023-10-09T04:23:35.266622+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:36.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:36 smithi082 bash[79733]: audit 2023-10-09T04:23:36.315709+0000 mon.a (mon.0) 442 : audit 0 from='client.? 172.21.15.82:0/1788756049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:38.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:38 smithi153 bash[59565]: cluster 2023-10-09T04:23:37.267494+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:38.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:38 smithi082 bash[77063]: cluster 2023-10-09T04:23:37.267494+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:38.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:38 smithi082 bash[79733]: cluster 2023-10-09T04:23:37.267494+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:39 smithi153 bash[59565]: audit 2023-10-09T04:23:38.681139+0000 mon.a (mon.0) 443 : audit 0 from='client.? 172.21.15.82:0/3985142829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:39.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:39 smithi082 bash[77063]: audit 2023-10-09T04:23:38.681139+0000 mon.a (mon.0) 443 : audit 0 from='client.? 172.21.15.82:0/3985142829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:39.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:39 smithi082 bash[79733]: audit 2023-10-09T04:23:38.681139+0000 mon.a (mon.0) 443 : audit 0 from='client.? 172.21.15.82:0/3985142829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:39] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.33.4" 2023-10-09T04:23:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:40 smithi153 bash[59565]: cluster 2023-10-09T04:23:39.268244+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:40 smithi153 bash[59565]: audit 2023-10-09T04:23:39.954438+0000 mon.a (mon.0) 444 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:40.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:40 smithi082 bash[77063]: cluster 2023-10-09T04:23:39.268244+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:40.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:40 smithi082 bash[77063]: audit 2023-10-09T04:23:39.954438+0000 mon.a (mon.0) 444 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:40.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:40 smithi082 bash[79733]: cluster 2023-10-09T04:23:39.268244+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:40.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:40 smithi082 bash[79733]: audit 2023-10-09T04:23:39.954438+0000 mon.a (mon.0) 444 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[77063]: audit 2023-10-09T04:23:39.954894+0000 mgr.y (mgr.34107) 266 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[77063]: cephadm 2023-10-09T04:23:39.957252+0000 mgr.y (mgr.34107) 267 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[77063]: audit 2023-10-09T04:23:41.032646+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.82:0/3239845808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[79733]: audit 2023-10-09T04:23:39.954894+0000 mgr.y (mgr.34107) 266 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[79733]: cephadm 2023-10-09T04:23:39.957252+0000 mgr.y (mgr.34107) 267 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T04:23:41.569 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:41 smithi082 bash[79733]: audit 2023-10-09T04:23:41.032646+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.82:0/3239845808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:41 smithi153 bash[59565]: audit 2023-10-09T04:23:39.954894+0000 mgr.y (mgr.34107) 266 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:23:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:41 smithi153 bash[59565]: cephadm 2023-10-09T04:23:39.957252+0000 mgr.y (mgr.34107) 267 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T04:23:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:41 smithi153 bash[59565]: audit 2023-10-09T04:23:41.032646+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.82:0/3239845808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:42 smithi153 bash[59565]: cluster 2023-10-09T04:23:41.269313+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:42 smithi153 bash[59565]: audit 2023-10-09T04:23:42.285482+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:42.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:42 smithi153 bash[59565]: audit 2023-10-09T04:23:42.287009+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:23:42.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:42 smithi153 bash[59565]: audit 2023-10-09T04:23:42.288293+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[79733]: cluster 2023-10-09T04:23:41.269313+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[79733]: audit 2023-10-09T04:23:42.285482+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[79733]: audit 2023-10-09T04:23:42.287009+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:23:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[79733]: audit 2023-10-09T04:23:42.288293+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:42.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[77063]: cluster 2023-10-09T04:23:41.269313+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:42.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[77063]: audit 2023-10-09T04:23:42.285482+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:42.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[77063]: audit 2023-10-09T04:23:42.287009+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:23:42.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:42 smithi082 bash[77063]: audit 2023-10-09T04:23:42.288293+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:43.629 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:43 smithi082 bash[77063]: cephadm 2023-10-09T04:23:42.277697+0000 mgr.y (mgr.34107) 269 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T04:23:43.629 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:43 smithi082 bash[77063]: cephadm 2023-10-09T04:23:42.291090+0000 mgr.y (mgr.34107) 270 : cephadm 1 Deploying daemon osd.3 on smithi082 2023-10-09T04:23:43.630 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:43 smithi082 bash[79733]: cephadm 2023-10-09T04:23:42.277697+0000 mgr.y (mgr.34107) 269 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T04:23:43.630 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:43 smithi082 bash[79733]: cephadm 2023-10-09T04:23:42.291090+0000 mgr.y (mgr.34107) 270 : cephadm 1 Deploying daemon osd.3 on smithi082 2023-10-09T04:23:43.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:43 smithi153 bash[59565]: cephadm 2023-10-09T04:23:42.277697+0000 mgr.y (mgr.34107) 269 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T04:23:43.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:43 smithi153 bash[59565]: cephadm 2023-10-09T04:23:42.291090+0000 mgr.y (mgr.34107) 270 : cephadm 1 Deploying daemon osd.3 on smithi082 2023-10-09T04:23:44.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:44 smithi153 bash[59565]: cluster 2023-10-09T04:23:43.270022+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:44.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:44 smithi153 bash[59565]: audit 2023-10-09T04:23:43.393852+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.82:0/1812634526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:44.685 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:44 smithi153 bash[59565]: audit 2023-10-09T04:23:44.250541+0000 mon.a (mon.0) 449 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[77063]: cluster 2023-10-09T04:23:43.270022+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[77063]: audit 2023-10-09T04:23:43.393852+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.82:0/1812634526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[77063]: audit 2023-10-09T04:23:44.250541+0000 mon.a (mon.0) 449 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[79733]: cluster 2023-10-09T04:23:43.270022+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[79733]: audit 2023-10-09T04:23:43.393852+0000 mon.a (mon.0) 448 : audit 0 from='client.? 172.21.15.82:0/1812634526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:44 smithi082 bash[79733]: audit 2023-10-09T04:23:44.250541+0000 mon.a (mon.0) 449 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:46.481 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:46 smithi082 systemd[1]: Stopping Ceph osd.3 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:23:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[77063]: cluster 2023-10-09T04:23:45.271053+0000 mgr.y (mgr.34107) 272 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[77063]: audit 2023-10-09T04:23:45.751807+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.82:0/3963687983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[79733]: cluster 2023-10-09T04:23:45.271053+0000 mgr.y (mgr.34107) 272 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[79733]: audit 2023-10-09T04:23:45.751807+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.82:0/3963687983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:46 smithi153 bash[59565]: cluster 2023-10-09T04:23:45.271053+0000 mgr.y (mgr.34107) 272 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:46 smithi153 bash[59565]: audit 2023-10-09T04:23:45.751807+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.82:0/3963687983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:46.982 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[49398]: debug 2023-10-09T04:23:46.580+0000 7f65f5070700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:23:46.982 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[49398]: debug 2023-10-09T04:23:46.580+0000 7f65f5070700 -1 osd.3 99 *** Got signal Terminated *** 2023-10-09T04:23:46.982 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:46 smithi082 bash[49398]: debug 2023-10-09T04:23:46.580+0000 7f65f5070700 -1 osd.3 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:23:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:47 smithi153 bash[59565]: cluster 2023-10-09T04:23:46.582845+0000 mon.a (mon.0) 451 : cluster 1 osd.3 marked itself down and dead 2023-10-09T04:23:47.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:47 smithi082 bash[77063]: cluster 2023-10-09T04:23:46.582845+0000 mon.a (mon.0) 451 : cluster 1 osd.3 marked itself down and dead 2023-10-09T04:23:47.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:47 smithi082 bash[79733]: cluster 2023-10-09T04:23:46.582845+0000 mon.a (mon.0) 451 : cluster 1 osd.3 marked itself down and dead 2023-10-09T04:23:48.286 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[93574]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T04:23:48.286 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[93667]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T04:23:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:48 smithi153 bash[59565]: cluster 2023-10-09T04:23:47.271880+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:48 smithi153 bash[59565]: cluster 2023-10-09T04:23:47.405933+0000 mon.a (mon.0) 452 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:48.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:48 smithi153 bash[59565]: cluster 2023-10-09T04:23:47.414754+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:48.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:48 smithi153 bash[59565]: cluster 2023-10-09T04:23:47.417378+0000 mon.a (mon.0) 453 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T04:23:48.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:48 smithi153 bash[59565]: audit 2023-10-09T04:23:48.104029+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.82:0/2322187892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[77063]: cluster 2023-10-09T04:23:47.271880+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[77063]: cluster 2023-10-09T04:23:47.405933+0000 mon.a (mon.0) 452 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[77063]: cluster 2023-10-09T04:23:47.414754+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[77063]: cluster 2023-10-09T04:23:47.417378+0000 mon.a (mon.0) 453 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[77063]: audit 2023-10-09T04:23:48.104029+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.82:0/2322187892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:48.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[79733]: cluster 2023-10-09T04:23:47.271880+0000 mgr.y (mgr.34107) 273 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[79733]: cluster 2023-10-09T04:23:47.405933+0000 mon.a (mon.0) 452 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[79733]: cluster 2023-10-09T04:23:47.414754+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[79733]: cluster 2023-10-09T04:23:47.417378+0000 mon.a (mon.0) 453 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T04:23:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:48 smithi082 bash[79733]: audit 2023-10-09T04:23:48.104029+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.82:0/2322187892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:47.415882+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.417678+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.420139+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.420295+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.420328+0000 mon.a (mon.0) 455 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T04:23:49.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.420891+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.420987+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.423710+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.424179+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[77063]: cluster 2023-10-09T04:23:48.426446+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:47.415882+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.417678+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.420139+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.420295+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.420328+0000 mon.a (mon.0) 455 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T04:23:49.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.420891+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.420987+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.423710+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.424179+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[79733]: cluster 2023-10-09T04:23:48.426446+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:47.415882+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.417678+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.420139+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.420295+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.420328+0000 mon.a (mon.0) 455 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.420891+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.420987+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.423710+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.424179+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.678 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:49 smithi153 bash[59565]: cluster 2023-10-09T04:23:48.426446+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T04:23:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:49] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.33.4" 2023-10-09T04:23:50.673 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:50 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T04:23:50.673 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:50 smithi082 systemd[1]: Stopped Ceph osd.3 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:50.673 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:50 smithi082 bash[77063]: cluster 2023-10-09T04:23:49.272682+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v210: 73 pgs: 10 stale+active+clean, 6 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:50.674 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:50 smithi082 bash[79733]: cluster 2023-10-09T04:23:49.272682+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v210: 73 pgs: 10 stale+active+clean, 6 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:50 smithi153 bash[59565]: cluster 2023-10-09T04:23:49.272682+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v210: 73 pgs: 10 stale+active+clean, 6 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:23:50.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:50 smithi082 systemd[1]: Started Ceph osd.3 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:51 smithi153 bash[59565]: audit 2023-10-09T04:23:50.464160+0000 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.82:0/1074277878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:51 smithi153 bash[59565]: audit 2023-10-09T04:23:50.710099+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:51 smithi153 bash[59565]: audit 2023-10-09T04:23:50.719452+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:51 smithi153 bash[59565]: audit 2023-10-09T04:23:50.724678+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:51.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[77063]: audit 2023-10-09T04:23:50.464160+0000 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.82:0/1074277878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:51.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[77063]: audit 2023-10-09T04:23:50.710099+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[77063]: audit 2023-10-09T04:23:50.719452+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[77063]: audit 2023-10-09T04:23:50.724678+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:51.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[79733]: audit 2023-10-09T04:23:50.464160+0000 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.82:0/1074277878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:51.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[79733]: audit 2023-10-09T04:23:50.710099+0000 mon.a (mon.0) 457 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[79733]: audit 2023-10-09T04:23:50.719452+0000 mon.a (mon.0) 458 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:51.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:51 smithi082 bash[79733]: audit 2023-10-09T04:23:50.724678+0000 mon.a (mon.0) 459 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:52 smithi153 bash[59565]: cluster 2023-10-09T04:23:51.273716+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v211: 73 pgs: 2 active+undersized+degraded, 4 stale+active+clean, 18 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:23:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:52 smithi153 bash[59565]: cluster 2023-10-09T04:23:51.716175+0000 mon.a (mon.0) 460 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:52.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[79733]: cluster 2023-10-09T04:23:51.273716+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v211: 73 pgs: 2 active+undersized+degraded, 4 stale+active+clean, 18 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:23:52.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[79733]: cluster 2023-10-09T04:23:51.716175+0000 mon.a (mon.0) 460 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:52.732 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:23:52.732 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:23:52.732 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T04:23:52.732 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:23:52.733 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:23:52.733 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T04:23:52.733 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:23:52.733 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[93846]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T04:23:52.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[77063]: cluster 2023-10-09T04:23:51.273716+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v211: 73 pgs: 2 active+undersized+degraded, 4 stale+active+clean, 18 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T04:23:52.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:52 smithi082 bash[77063]: cluster 2023-10-09T04:23:51.716175+0000 mon.a (mon.0) 460 : cluster 3 Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:53 smithi153 bash[59565]: audit 2023-10-09T04:23:52.811528+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.82:0/598672626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:53.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:53 smithi082 bash[79733]: audit 2023-10-09T04:23:52.811528+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.82:0/598672626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:53.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:53 smithi082 bash[77063]: audit 2023-10-09T04:23:52.811528+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.82:0/598672626' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:54.232 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:53 smithi082 bash[94164]: debug 2023-10-09T04:23:53.964+0000 7f2955d0b700 1 -- 172.21.15.82:0/748971529 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555a3b3f04e0 con 0x555a3b44b000 2023-10-09T04:23:54.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:54 smithi082 bash[79733]: cluster 2023-10-09T04:23:53.274695+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:54.732 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:54 smithi082 bash[94164]: debug 2023-10-09T04:23:54.508+0000 7f295ee91640 -1 Falling back to public interface 2023-10-09T04:23:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:54 smithi082 bash[77063]: cluster 2023-10-09T04:23:53.274695+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:54 smithi153 bash[59565]: cluster 2023-10-09T04:23:53.274695+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:55.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:55 smithi082 bash[77063]: audit 2023-10-09T04:23:55.184733+0000 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.82:0/2797158444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:55.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:55 smithi082 bash[79733]: audit 2023-10-09T04:23:55.184733+0000 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.82:0/2797158444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:55 smithi153 bash[59565]: audit 2023-10-09T04:23:55.184733+0000 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.82:0/2797158444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:56.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:56 smithi082 bash[77063]: cluster 2023-10-09T04:23:55.275693+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v213: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:56.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:56 smithi082 bash[79733]: cluster 2023-10-09T04:23:55.275693+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v213: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:56 smithi153 bash[59565]: cluster 2023-10-09T04:23:55.275693+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v213: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: audit 2023-10-09T04:23:56.765809+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: audit 2023-10-09T04:23:56.772896+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: cluster 2023-10-09T04:23:57.276188+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: audit 2023-10-09T04:23:57.544284+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.82:0/689686606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: audit 2023-10-09T04:23:57.570026+0000 mon.a (mon.0) 466 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:57 smithi153 bash[59565]: audit 2023-10-09T04:23:57.576506+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.232 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[94164]: debug 2023-10-09T04:23:57.796+0000 7f295ee91640 -1 osd.3 99 log_to_monitors true 2023-10-09T04:23:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: audit 2023-10-09T04:23:56.765809+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: audit 2023-10-09T04:23:56.772896+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: cluster 2023-10-09T04:23:57.276188+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: audit 2023-10-09T04:23:57.544284+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.82:0/689686606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: audit 2023-10-09T04:23:57.570026+0000 mon.a (mon.0) 466 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[79733]: audit 2023-10-09T04:23:57.576506+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: audit 2023-10-09T04:23:56.765809+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: audit 2023-10-09T04:23:56.772896+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: cluster 2023-10-09T04:23:57.276188+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: audit 2023-10-09T04:23:57.544284+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.82:0/689686606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: audit 2023-10-09T04:23:57.570026+0000 mon.a (mon.0) 466 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:57 smithi082 bash[77063]: audit 2023-10-09T04:23:57.576506+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:59.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:58 smithi153 bash[59565]: audit 2023-10-09T04:23:57.802647+0000 mon.a (mon.0) 468 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:23:59.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:58 smithi082 bash[77063]: audit 2023-10-09T04:23:57.802647+0000 mon.a (mon.0) 468 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:23:59.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:58 smithi082 bash[79733]: audit 2023-10-09T04:23:57.802647+0000 mon.a (mon.0) 468 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:23:59.232 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 04:23:58 smithi082 bash[94164]: debug 2023-10-09T04:23:58.816+0000 7f294caf6700 -1 osd.3 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:23:59.956 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:23:59] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: audit 2023-10-09T04:23:58.784588+0000 mon.a (mon.0) 469 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:58.784695+0000 mon.a (mon.0) 470 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: audit 2023-10-09T04:23:58.786313+0000 mon.a (mon.0) 471 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:58.800930+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:58.801021+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:58.801036+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:58.801121+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.244237+0000 mon.a (mon.0) 472 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: audit 2023-10-09T04:23:59.256114+0000 mon.a (mon.0) 473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: audit 2023-10-09T04:23:59.257511+0000 mon.a (mon.0) 474 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.276755+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v216: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[77063]: audit 2023-10-09T04:23:59.369525+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: audit 2023-10-09T04:23:58.784588+0000 mon.a (mon.0) 469 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:58.784695+0000 mon.a (mon.0) 470 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: audit 2023-10-09T04:23:58.786313+0000 mon.a (mon.0) 471 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:23:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:58.800930+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:58.801021+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:58.801036+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:58.801121+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.244237+0000 mon.a (mon.0) 472 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: audit 2023-10-09T04:23:59.256114+0000 mon.a (mon.0) 473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: audit 2023-10-09T04:23:59.257511+0000 mon.a (mon.0) 474 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.276755+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v216: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:23:59.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:23:59 smithi082 bash[79733]: audit 2023-10-09T04:23:59.369525+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: audit 2023-10-09T04:23:58.784588+0000 mon.a (mon.0) 469 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:58.784695+0000 mon.a (mon.0) 470 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: audit 2023-10-09T04:23:58.786313+0000 mon.a (mon.0) 471 : audit 1 from='osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:58.800930+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:58.801021+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:58.801036+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:58.801121+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.244237+0000 mon.a (mon.0) 472 : cluster 3 Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:00.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: audit 2023-10-09T04:23:59.256114+0000 mon.a (mon.0) 473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: audit 2023-10-09T04:23:59.257511+0000 mon.a (mon.0) 474 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.276755+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v216: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T04:24:00.178 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:23:59 smithi153 bash[59565]: audit 2023-10-09T04:23:59.369525+0000 mon.a (mon.0) 475 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:00.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:24:01.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.782163+0000 mon.a (mon.0) 476 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.784913+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.785643+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.789141+0000 mon.a (mon.0) 477 : cluster 1 osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592] boot 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: cluster 2023-10-09T04:23:59.789236+0000 mon.a (mon.0) 478 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: audit 2023-10-09T04:23:59.789798+0000 mon.a (mon.0) 479 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:24:01.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:00 smithi153 bash[59565]: audit 2023-10-09T04:23:59.936814+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.82:0/1098975722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.782163+0000 mon.a (mon.0) 476 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.784913+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.785643+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.789141+0000 mon.a (mon.0) 477 : cluster 1 osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592] boot 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: cluster 2023-10-09T04:23:59.789236+0000 mon.a (mon.0) 478 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: audit 2023-10-09T04:23:59.789798+0000 mon.a (mon.0) 479 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[77063]: audit 2023-10-09T04:23:59.936814+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.82:0/1098975722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:01.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.782163+0000 mon.a (mon.0) 476 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.784913+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.785643+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e103 with expected crc 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.789141+0000 mon.a (mon.0) 477 : cluster 1 osd.3 [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592] boot 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: cluster 2023-10-09T04:23:59.789236+0000 mon.a (mon.0) 478 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: audit 2023-10-09T04:23:59.789798+0000 mon.a (mon.0) 479 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:24:01.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:00 smithi082 bash[79733]: audit 2023-10-09T04:23:59.936814+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.82:0/1098975722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.793483+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.798086+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.798786+0000 mon.a (mon.0) 481 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.800425+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.800973+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.801567+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.801748+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.802636+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.804575+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.804893+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.806090+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:00.808535+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[79733]: cluster 2023-10-09T04:24:01.277477+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v219: 73 pgs: 8 peering, 2 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.793483+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.798086+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.165 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.798786+0000 mon.a (mon.0) 481 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.800425+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.800973+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.801567+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.801748+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.802636+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.804575+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.804893+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.806090+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:00.808535+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:01 smithi082 bash[77063]: cluster 2023-10-09T04:24:01.277477+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v219: 73 pgs: 8 peering, 2 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.793483+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.798086+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.798786+0000 mon.a (mon.0) 481 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.800425+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.800973+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.801567+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.801748+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.802636+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.804575+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.804893+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.806090+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:00.808535+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T04:24:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:01 smithi153 bash[59565]: cluster 2023-10-09T04:24:01.277477+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v219: 73 pgs: 8 peering, 2 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:04.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:04 smithi082 bash[77063]: cluster 2023-10-09T04:24:03.278081+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v220: 73 pgs: 8 peering, 2 active+undersized+degraded, 14 active+undersized, 49 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:04.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:04 smithi082 bash[77063]: cluster 2023-10-09T04:24:04.245957+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:04.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:04 smithi082 bash[79733]: cluster 2023-10-09T04:24:03.278081+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v220: 73 pgs: 8 peering, 2 active+undersized+degraded, 14 active+undersized, 49 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:04.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:04 smithi082 bash[79733]: cluster 2023-10-09T04:24:04.245957+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:04.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:04 smithi153 bash[59565]: cluster 2023-10-09T04:24:03.278081+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v220: 73 pgs: 8 peering, 2 active+undersized+degraded, 14 active+undersized, 49 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:24:04.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:04 smithi153 bash[59565]: cluster 2023-10-09T04:24:04.245957+0000 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:04.856 INFO:teuthology.orchestra.run.smithi082.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:24:06.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:05 smithi153 bash[59565]: audit 2023-10-09T04:24:04.847886+0000 mgr.y (mgr.34107) 282 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:06.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:05 smithi153 bash[59565]: cephadm 2023-10-09T04:24:04.848284+0000 mgr.y (mgr.34107) 283 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:24:06.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:05 smithi153 bash[59565]: audit 2023-10-09T04:24:04.854511+0000 mon.a (mon.0) 483 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:06.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:05 smithi153 bash[59565]: cluster 2023-10-09T04:24:05.279189+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:06.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[77063]: audit 2023-10-09T04:24:04.847886+0000 mgr.y (mgr.34107) 282 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:06.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[77063]: cephadm 2023-10-09T04:24:04.848284+0000 mgr.y (mgr.34107) 283 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:24:06.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[77063]: audit 2023-10-09T04:24:04.854511+0000 mon.a (mon.0) 483 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:06.203 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[77063]: cluster 2023-10-09T04:24:05.279189+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:06.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[79733]: audit 2023-10-09T04:24:04.847886+0000 mgr.y (mgr.34107) 282 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:06.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[79733]: cephadm 2023-10-09T04:24:04.848284+0000 mgr.y (mgr.34107) 283 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:24:06.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[79733]: audit 2023-10-09T04:24:04.854511+0000 mon.a (mon.0) 483 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:06.204 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:05 smithi082 bash[79733]: cluster 2023-10-09T04:24:05.279189+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:06.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:24:07.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:06 smithi082 bash[77063]: cluster 2023-10-09T04:24:05.850489+0000 mon.a (mon.0) 484 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T04:24:07.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:06 smithi082 bash[77063]: cluster 2023-10-09T04:24:05.850515+0000 mon.a (mon.0) 485 : cluster 1 Cluster is now healthy 2023-10-09T04:24:07.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:06 smithi082 bash[79733]: cluster 2023-10-09T04:24:05.850489+0000 mon.a (mon.0) 484 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T04:24:07.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:06 smithi082 bash[79733]: cluster 2023-10-09T04:24:05.850515+0000 mon.a (mon.0) 485 : cluster 1 Cluster is now healthy 2023-10-09T04:24:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:06 smithi153 bash[59565]: cluster 2023-10-09T04:24:05.850489+0000 mon.a (mon.0) 484 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T04:24:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:06 smithi153 bash[59565]: cluster 2023-10-09T04:24:05.850515+0000 mon.a (mon.0) 485 : cluster 1 Cluster is now healthy 2023-10-09T04:24:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:07 smithi153 bash[59565]: cluster 2023-10-09T04:24:07.279985+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:08.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:07 smithi082 bash[79733]: cluster 2023-10-09T04:24:07.279985+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:07 smithi082 bash[77063]: cluster 2023-10-09T04:24:07.279985+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:09.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.323533+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.331015+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.332419+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.333617+0000 mon.a (mon.0) 489 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.340301+0000 mon.a (mon.0) 490 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.422625+0000 mon.a (mon.0) 491 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.425614+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.426852+0000 mon.a (mon.0) 493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[79733]: audit 2023-10-09T04:24:08.434319+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.642 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.323533+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.331015+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.332419+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.333617+0000 mon.a (mon.0) 489 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.340301+0000 mon.a (mon.0) 490 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.422625+0000 mon.a (mon.0) 491 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.425614+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.426852+0000 mon.a (mon.0) 493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.643 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[77063]: audit 2023-10-09T04:24:08.434319+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.323533+0000 mon.a (mon.0) 486 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.331015+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.332419+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.333617+0000 mon.a (mon.0) 489 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.340301+0000 mon.a (mon.0) 490 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.422625+0000 mon.a (mon.0) 491 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.425614+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.426852+0000 mon.a (mon.0) 493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:09.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:09 smithi153 bash[59565]: audit 2023-10-09T04:24:08.434319+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:09] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-09T04:24:10.136 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (8m) - 14m 13.4M - 0.23.0 ba2b418f427c 4994b766acec 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (8m) - 14m 41.3M - 8.3.5 dad864ee21e9 24b93cfba910 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (12m) - 12m 17.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (12m) - 12m 14.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443,9283,8765 running (7m) - 21m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 b4d6262d244c 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:8443,9283,8765 running (7m) - 23m 482M - 18.2.0-598-gb4fa663f 3060897fdf97 588466e4b0b0 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (6m) - 23m 74.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 93a348625d31 2023-10-09T04:24:10.137 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 running (3m) - 21m 39.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 41a655ab0b3c 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 running (5m) - 21m 47.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 27025ca926cb 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (8m) - 14m 9180k - 1.3.1 1dbe0e931976 cda947bde3d9 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (8m) - 14m 9096k - 1.3.1 1dbe0e931976 93816ab22ea5 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (114s) - 20m 43.6M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 dd19d41daea7 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (87s) - 19m 42.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 65018d84fe86 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (59s) - 19m 40.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 76f9d57d420c 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (17s) - 18m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 e7550f2353e7 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (17m) - 17m 49.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (17m) - 17m 46.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:24:10.138 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (16m) - 16m 48.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:24:10.139 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (15m) - 15m 52.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:24:10.139 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (7m) - 14m 59.3M - 2.33.4 514e6a882f6e be2d59b8aa70 2023-10-09T04:24:10.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:10 smithi082 bash[79733]: cluster 2023-10-09T04:24:09.280805+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:10.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:10 smithi082 bash[77063]: cluster 2023-10-09T04:24:09.280805+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:10.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:10 smithi153 bash[59565]: cluster 2023-10-09T04:24:09.280805+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:10.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:24:11.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:11 smithi082 bash[79733]: audit 2023-10-09T04:24:10.116129+0000 mgr.y (mgr.34107) 287 : audit 0 from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:11.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:11 smithi082 bash[77063]: audit 2023-10-09T04:24:10.116129+0000 mgr.y (mgr.34107) 287 : audit 0 from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:11.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:11 smithi153 bash[59565]: audit 2023-10-09T04:24:10.116129+0000 mgr.y (mgr.34107) 287 : audit 0 from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:12 smithi153 bash[59565]: cluster 2023-10-09T04:24:11.281783+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:12 smithi082 bash[77063]: cluster 2023-10-09T04:24:11.281783+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:12.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:12 smithi082 bash[79733]: cluster 2023-10-09T04:24:11.281783+0000 mgr.y (mgr.34107) 288 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.a smithi082 *:9093,9094 running (8m) - 14m 13.4M - 0.23.0 ba2b418f427c 4994b766acec 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.a smithi153 *:3000 running (8m) - 14m 41.3M - 8.3.5 dad864ee21e9 24b93cfba910 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi082.juwitc smithi082 running (12m) - 12m 17.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d67fccbfbd3c 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foo.smithi153.braqax smithi153 running (12m) - 12m 14.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8705ea49c69f 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.x smithi153 *:8443,9283,8765 running (7m) - 21m 428M - 18.2.0-598-gb4fa663f 3060897fdf97 b4d6262d244c 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.y smithi082 *:8443,9283,8765 running (8m) - 23m 482M - 18.2.0-598-gb4fa663f 3060897fdf97 588466e4b0b0 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a smithi082 running (6m) - 23m 74.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 93a348625d31 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b smithi153 running (3m) - 21m 39.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 41a655ab0b3c 2023-10-09T04:24:14.174 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c smithi082 running (5m) - 21m 47.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 27025ca926cb 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.a smithi082 *:9100 running (8m) - 14m 9180k - 1.3.1 1dbe0e931976 cda947bde3d9 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.b smithi153 *:9100 running (8m) - 14m 9096k - 1.3.1 1dbe0e931976 93816ab22ea5 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (118s) - 20m 43.6M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 dd19d41daea7 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (91s) - 19m 42.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 65018d84fe86 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (63s) - 19m 40.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 76f9d57d420c 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (21s) - 18m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 e7550f2353e7 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi153 running (17m) - 17m 49.7M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 beb8b56b8204 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi153 running (17m) - 17m 46.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d50369a483aa 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi153 running (16m) - 16m 48.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e8c8ae827715 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi153 running (15m) - 15m 52.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 331e0fde02a2 2023-10-09T04:24:14.175 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.a smithi153 *:9095 running (7m) - 14m 59.3M - 2.33.4 514e6a882f6e be2d59b8aa70 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[79733]: cluster 2023-10-09T04:24:13.282586+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[79733]: audit 2023-10-09T04:24:14.263805+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[79733]: audit 2023-10-09T04:24:14.265132+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[77063]: cluster 2023-10-09T04:24:13.282586+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[77063]: audit 2023-10-09T04:24:14.263805+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:14 smithi082 bash[77063]: audit 2023-10-09T04:24:14.265132+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:14 smithi153 bash[59565]: cluster 2023-10-09T04:24:13.282586+0000 mgr.y (mgr.34107) 289 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:14 smithi153 bash[59565]: audit 2023-10-09T04:24:14.263805+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:14 smithi153 bash[59565]: audit 2023-10-09T04:24:14.265132+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:14.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T04:24:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:15 smithi153 bash[59565]: audit 2023-10-09T04:24:14.154653+0000 mgr.y (mgr.34107) 290 : audit 0 from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:15 smithi153 bash[59565]: audit 2023-10-09T04:24:14.384212+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:15 smithi082 bash[77063]: audit 2023-10-09T04:24:14.154653+0000 mgr.y (mgr.34107) 290 : audit 0 from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:15 smithi082 bash[77063]: audit 2023-10-09T04:24:14.384212+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:15 smithi082 bash[79733]: audit 2023-10-09T04:24:14.154653+0000 mgr.y (mgr.34107) 290 : audit 0 from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:24:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:15 smithi082 bash[79733]: audit 2023-10-09T04:24:14.384212+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:24:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:16 smithi153 bash[59565]: cluster 2023-10-09T04:24:15.283605+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:16.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:16 smithi082 bash[79733]: cluster 2023-10-09T04:24:15.283605+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:16.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:16 smithi082 bash[77063]: cluster 2023-10-09T04:24:15.283605+0000 mgr.y (mgr.34107) 291 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:18.434 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:24:18.434 INFO:teuthology.orchestra.run.smithi082.stdout: "mon": { 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr": { 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd": { 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T04:24:18.435 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: "mds": { 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: "overall": { 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:24:18.436 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:24:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:18 smithi153 bash[59565]: cluster 2023-10-09T04:24:17.284493+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:18.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:18 smithi082 bash[77063]: cluster 2023-10-09T04:24:17.284493+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:18.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:18 smithi082 bash[79733]: cluster 2023-10-09T04:24:17.284493+0000 mgr.y (mgr.34107) 292 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:19.209 DEBUG:teuthology.parallel:result is None 2023-10-09T04:24:19.210 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:24:19.219 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T04:24:19.219 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:24:19.239 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:24:19.239 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:24:19.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:24:19.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:19 smithi082 bash[79733]: audit 2023-10-09T04:24:18.432534+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.82:0/3876934021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:19.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:19 smithi082 bash[77063]: audit 2023-10-09T04:24:18.432534+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.82:0/3876934021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:19 smithi153 bash[59565]: audit 2023-10-09T04:24:18.432534+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.82:0/3876934021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:19.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:19] "GET /metrics HTTP/1.1" 200 33886 "" "Prometheus/2.33.4" 2023-10-09T04:24:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:20 smithi153 bash[59565]: cluster 2023-10-09T04:24:19.285342+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:20 smithi082 bash[77063]: cluster 2023-10-09T04:24:19.285342+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:20.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:20 smithi082 bash[79733]: cluster 2023-10-09T04:24:19.285342+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:22 smithi153 bash[59565]: cluster 2023-10-09T04:24:21.286182+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:22.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:22 smithi082 bash[77063]: cluster 2023-10-09T04:24:21.286182+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:22 smithi082 bash[79733]: cluster 2023-10-09T04:24:21.286182+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:24.021 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:24 smithi153 bash[59565]: cluster 2023-10-09T04:24:23.286933+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:24.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:24 smithi082 bash[79733]: cluster 2023-10-09T04:24:23.286933+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:24.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:24 smithi082 bash[77063]: cluster 2023-10-09T04:24:23.286933+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:26 smithi153 bash[59565]: cluster 2023-10-09T04:24:25.287878+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:26.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:26 smithi082 bash[79733]: cluster 2023-10-09T04:24:25.287878+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:26.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:26 smithi082 bash[77063]: cluster 2023-10-09T04:24:25.287878+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:27.230 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:24:27.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:27 smithi153 bash[59565]: audit 2023-10-09T04:24:27.218438+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.82:0/1294720576' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:24:27.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:27 smithi082 bash[77063]: audit 2023-10-09T04:24:27.218438+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.82:0/1294720576' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:24:27.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:27 smithi082 bash[79733]: audit 2023-10-09T04:24:27.218438+0000 mon.a (mon.0) 498 : audit 0 from='client.? 172.21.15.82:0/1294720576' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:24:28.030 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":35,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6800","nonce":4029935854},{"type":"v1","addr":"172.21.15.82:6801","nonce":4029935854}]},"active_addr":"172.21.15.82:6801/4029935854","active_change":"2023-10-09T04:17:29.105556+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25597,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.82:8443/","prometheus":"http://172.21.15.82:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":84,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":189782451}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":92107566}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":1375661432}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":2366588282}]}]} 2023-10-09T04:24:28.034 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:24:28.035 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:24:28.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:24:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:28 smithi153 bash[59565]: cluster 2023-10-09T04:24:27.288536+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:28.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:28 smithi082 bash[77063]: cluster 2023-10-09T04:24:27.288536+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:28.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:28 smithi082 bash[79733]: cluster 2023-10-09T04:24:27.288536+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:29 smithi153 bash[59565]: audit 2023-10-09T04:24:29.251898+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:29 smithi082 bash[77063]: audit 2023-10-09T04:24:29.251898+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:29 smithi082 bash[79733]: audit 2023-10-09T04:24:29.251898+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:29] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T04:24:30.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:30 smithi082 bash[77063]: cluster 2023-10-09T04:24:29.289299+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:30 smithi082 bash[79733]: cluster 2023-10-09T04:24:29.289299+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:30 smithi153 bash[59565]: cluster 2023-10-09T04:24:29.289299+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:32 smithi082 bash[77063]: cluster 2023-10-09T04:24:31.289914+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:32 smithi082 bash[79733]: cluster 2023-10-09T04:24:31.289914+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:32.810 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:32.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:32 smithi153 bash[59565]: cluster 2023-10-09T04:24:31.289914+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:34.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:34 smithi082 bash[79733]: cluster 2023-10-09T04:24:33.290765+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:34.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:34 smithi082 bash[77063]: cluster 2023-10-09T04:24:33.290765+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:34.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:34 smithi153 bash[59565]: cluster 2023-10-09T04:24:33.290765+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:35.967 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:24:35.968 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":104,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:24:00.790711+0000","last_up_change":"2023-10-09T04:23:59.781409+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6803","nonce":85623288}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6805","nonce":85623288}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6809","nonce":85623288}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6807","nonce":85623288}]},"public_addr":"172.21.15.82:6803/85623288","cluster_addr":"172.21.15.82:6805/85623288","heartbeat_back_addr":"172.21.15.82:6809/85623288","heartbeat_front_addr":"172.21.15.82:6807/85623288","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6811","nonce":1232825898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6813","nonce":1232825898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6817","nonce":1232825898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6815","nonce":1232825898}]},"public_addr":"172.21.15.82:6811/1232825898","cluster_addr":"172.21.15.82:6813/1232825898","heartbeat_back_addr":"172.21.15.82:6817/1232825898","heartbeat_front_addr":"172.21.15.82:6815/1232825898","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6819","nonce":2407238682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6821","nonce":2407238682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6825","nonce":2407238682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6823","nonce":2407238682}]},"public_addr":"172.21.15.82:6819/2407238682","cluster_addr":"172.21.15.82:6821/2407238682","heartbeat_back_addr":"172.21.15.82:6825/2407238682","heartbeat_front_addr":"172.21.15.82:6823/2407238682","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6827","nonce":3740620592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6829","nonce":3740620592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6833","nonce":3740620592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6831","nonce":3740620592}]},"public_addr":"172.21.15.82:6827/3740620592","cluster_addr":"172.21.15.82:6829/3740620592","heartbeat_back_addr":"172.21.15.82:6833/3740620592","heartbeat_front_addr":"172.21.15.82:6831/3740620592","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/3203316964":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/3169418397":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/590497495":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6801/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/2018378994":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1290610280":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.153:6834/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/2959459015":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1415425338":"2023-10-10T04:16:27.265506+0000","172.21.15.153:6835/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/3596291160":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/3909792950":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/2145280144":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000","172.21.15.153:0/1754741933":"2023-10-10T04:16:27.265506+0000","172.21.15.153:0/3564317560":"2023-10-10T04:16:27.265506+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:24:36.697 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:36 smithi082 bash[77063]: cluster 2023-10-09T04:24:35.291860+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:36.697 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:36 smithi082 bash[77063]: audit 2023-10-09T04:24:35.960113+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.82:0/2642589516' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:36.697 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:36 smithi082 bash[79733]: cluster 2023-10-09T04:24:35.291860+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:36.697 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:36 smithi082 bash[79733]: audit 2023-10-09T04:24:35.960113+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.82:0/2642589516' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:36.699 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:24:36.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:24:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:36 smithi153 bash[59565]: cluster 2023-10-09T04:24:35.291860+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:36 smithi153 bash[59565]: audit 2023-10-09T04:24:35.960113+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.82:0/2642589516' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:38.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:38 smithi082 bash[79733]: cluster 2023-10-09T04:24:37.292612+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:38.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:38 smithi082 bash[77063]: cluster 2023-10-09T04:24:37.292612+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:38.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:38 smithi153 bash[59565]: cluster 2023-10-09T04:24:37.292612+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:39] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T04:24:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:40 smithi153 bash[59565]: cluster 2023-10-09T04:24:39.293428+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:40 smithi082 bash[77063]: cluster 2023-10-09T04:24:39.293428+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:40.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:40 smithi082 bash[79733]: cluster 2023-10-09T04:24:39.293428+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:41.480 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:42.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:42 smithi153 bash[59565]: cluster 2023-10-09T04:24:41.294326+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:42.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:42 smithi082 bash[77063]: cluster 2023-10-09T04:24:41.294326+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:42.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:42 smithi082 bash[79733]: cluster 2023-10-09T04:24:41.294326+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:44.509 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:24:44.509 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":104,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T04:24:00.790711+0000","last_up_change":"2023-10-09T04:23:59.781409+0000","last_in_change":"2023-10-09T04:08:10.303522+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6803","nonce":85623288}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6805","nonce":85623288}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6809","nonce":85623288}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6807","nonce":85623288}]},"public_addr":"172.21.15.82:6803/85623288","cluster_addr":"172.21.15.82:6805/85623288","heartbeat_back_addr":"172.21.15.82:6809/85623288","heartbeat_front_addr":"172.21.15.82:6807/85623288","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6811","nonce":1232825898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6813","nonce":1232825898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6817","nonce":1232825898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6815","nonce":1232825898}]},"public_addr":"172.21.15.82:6811/1232825898","cluster_addr":"172.21.15.82:6813/1232825898","heartbeat_back_addr":"172.21.15.82:6817/1232825898","heartbeat_front_addr":"172.21.15.82:6815/1232825898","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6819","nonce":2407238682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6821","nonce":2407238682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6825","nonce":2407238682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2407238682},{"type":"v1","addr":"172.21.15.82:6823","nonce":2407238682}]},"public_addr":"172.21.15.82:6819/2407238682","cluster_addr":"172.21.15.82:6821/2407238682","heartbeat_back_addr":"172.21.15.82:6825/2407238682","heartbeat_front_addr":"172.21.15.82:6823/2407238682","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6827","nonce":3740620592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6829","nonce":3740620592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6833","nonce":3740620592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6831","nonce":3740620592}]},"public_addr":"172.21.15.82:6827/3740620592","cluster_addr":"172.21.15.82:6829/3740620592","heartbeat_back_addr":"172.21.15.82:6833/3740620592","heartbeat_front_addr":"172.21.15.82:6831/3740620592","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6809","nonce":1288836071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6811","nonce":1288836071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6815","nonce":1288836071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":1288836071},{"type":"v1","addr":"172.21.15.153:6813","nonce":1288836071}]},"public_addr":"172.21.15.153:6809/1288836071","cluster_addr":"172.21.15.153:6811/1288836071","heartbeat_back_addr":"172.21.15.153:6815/1288836071","heartbeat_front_addr":"172.21.15.153:6813/1288836071","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6817","nonce":2167557505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6819","nonce":2167557505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6823","nonce":2167557505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":2167557505},{"type":"v1","addr":"172.21.15.153:6821","nonce":2167557505}]},"public_addr":"172.21.15.153:6817/2167557505","cluster_addr":"172.21.15.153:6819/2167557505","heartbeat_back_addr":"172.21.15.153:6823/2167557505","heartbeat_front_addr":"172.21.15.153:6821/2167557505","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6825","nonce":3367585403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6827","nonce":3367585403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6831","nonce":3367585403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3367585403},{"type":"v1","addr":"172.21.15.153:6829","nonce":3367585403}]},"public_addr":"172.21.15.153:6825/3367585403","cluster_addr":"172.21.15.153:6827/3367585403","heartbeat_back_addr":"172.21.15.153:6831/3367585403","heartbeat_front_addr":"172.21.15.153:6829/3367585403","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/3203316964":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/3169418397":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/590497495":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6801/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/2018378994":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1290610280":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.153:6834/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/2959459015":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1415425338":"2023-10-10T04:16:27.265506+0000","172.21.15.153:6835/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/3596291160":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/3909792950":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/2145280144":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000","172.21.15.153:0/1754741933":"2023-10-10T04:16:27.265506+0000","172.21.15.153:0/3564317560":"2023-10-10T04:16:27.265506+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:24:44.666 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:44 smithi082 bash[77063]: cluster 2023-10-09T04:24:43.294846+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:44.666 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:44 smithi082 bash[77063]: audit 2023-10-09T04:24:44.252514+0000 mon.a (mon.0) 500 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:44.666 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:44 smithi082 bash[79733]: cluster 2023-10-09T04:24:43.294846+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:44.666 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:44 smithi082 bash[79733]: audit 2023-10-09T04:24:44.252514+0000 mon.a (mon.0) 500 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:44 smithi153 bash[59565]: cluster 2023-10-09T04:24:43.294846+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:44 smithi153 bash[59565]: audit 2023-10-09T04:24:44.252514+0000 mon.a (mon.0) 500 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:45.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:24:45.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:24:45.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:24:45.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:24:45.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:24:45.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:24:45.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:24:45.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:24:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:45 smithi153 bash[59565]: audit 2023-10-09T04:24:44.504199+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.82:0/580952376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:45 smithi082 bash[77063]: audit 2023-10-09T04:24:44.504199+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.82:0/580952376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:45 smithi082 bash[79733]: audit 2023-10-09T04:24:44.504199+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.82:0/580952376' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:24:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:46 smithi153 bash[59565]: cluster 2023-10-09T04:24:45.295588+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:46 smithi082 bash[77063]: cluster 2023-10-09T04:24:45.295588+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:46 smithi082 bash[79733]: cluster 2023-10-09T04:24:45.295588+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:48 smithi153 bash[59565]: cluster 2023-10-09T04:24:47.296279+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:48.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:48 smithi082 bash[77063]: cluster 2023-10-09T04:24:47.296279+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:48.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:48 smithi082 bash[79733]: cluster 2023-10-09T04:24:47.296279+0000 mgr.y (mgr.34107) 307 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:49] "GET /metrics HTTP/1.1" 200 33886 "" "Prometheus/2.33.4" 2023-10-09T04:24:50.142 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.143 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.143 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.143 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.144 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.144 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.149 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.150 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:50 smithi082 bash[77063]: cluster 2023-10-09T04:24:49.297091+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:50 smithi082 bash[79733]: cluster 2023-10-09T04:24:49.297091+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:50 smithi153 bash[59565]: cluster 2023-10-09T04:24:49.297091+0000 mgr.y (mgr.34107) 308 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:52 smithi153 bash[59565]: cluster 2023-10-09T04:24:51.297630+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:52.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:52 smithi082 bash[79733]: cluster 2023-10-09T04:24:51.297630+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:52.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:52 smithi082 bash[77063]: cluster 2023-10-09T04:24:51.297630+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:54 smithi082 bash[77063]: cluster 2023-10-09T04:24:53.298138+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:54 smithi082 bash[79733]: cluster 2023-10-09T04:24:53.298138+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:54 smithi153 bash[59565]: cluster 2023-10-09T04:24:53.298138+0000 mgr.y (mgr.34107) 310 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:56.418 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:56 smithi082 bash[77063]: cluster 2023-10-09T04:24:55.298997+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:56.418 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:56 smithi082 bash[79733]: cluster 2023-10-09T04:24:55.298997+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:56 smithi153 bash[59565]: cluster 2023-10-09T04:24:55.298997+0000 mgr.y (mgr.34107) 311 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:58.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:58 smithi082 bash[79733]: cluster 2023-10-09T04:24:57.299712+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:58.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:58 smithi082 bash[77063]: cluster 2023-10-09T04:24:57.299712+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:58 smithi153 bash[59565]: cluster 2023-10-09T04:24:57.299712+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:24:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:24:59 smithi153 bash[59565]: audit 2023-10-09T04:24:59.252818+0000 mon.a (mon.0) 501 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:24:59 smithi082 bash[77063]: audit 2023-10-09T04:24:59.252818+0000 mon.a (mon.0) 501 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:24:59 smithi082 bash[79733]: audit 2023-10-09T04:24:59.252818+0000 mon.a (mon.0) 501 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:24:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:24:59] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:25:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:00 smithi153 bash[59565]: cluster 2023-10-09T04:24:59.300559+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:00.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:00 smithi082 bash[77063]: cluster 2023-10-09T04:24:59.300559+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:00.714 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:00 smithi082 bash[79733]: cluster 2023-10-09T04:24:59.300559+0000 mgr.y (mgr.34107) 313 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:02 smithi153 bash[59565]: cluster 2023-10-09T04:25:01.301215+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:02.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:02 smithi082 bash[77063]: cluster 2023-10-09T04:25:01.301215+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:02.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:02 smithi082 bash[79733]: cluster 2023-10-09T04:25:01.301215+0000 mgr.y (mgr.34107) 314 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:03.191 INFO:teuthology.orchestra.run.smithi082.stdout:206158430407 2023-10-09T04:25:03.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:25:04.017 INFO:teuthology.orchestra.run.smithi082.stdout:180388626640 2023-10-09T04:25:04.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:25:04.548 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:04 smithi082 bash[79733]: cluster 2023-10-09T04:25:03.302067+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:04.548 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:04 smithi082 bash[77063]: cluster 2023-10-09T04:25:03.302067+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:04.627 INFO:teuthology.orchestra.run.smithi082.stdout:154618822871 2023-10-09T04:25:04.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:25:04.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:04 smithi153 bash[59565]: cluster 2023-10-09T04:25:03.302067+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:06.388 INFO:teuthology.orchestra.run.smithi082.stdout:420906795031 2023-10-09T04:25:06.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:25:06.514 INFO:teuthology.orchestra.run.smithi082.stdout:442381631502 2023-10-09T04:25:06.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:25:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:06 smithi153 bash[59565]: cluster 2023-10-09T04:25:05.302639+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:06 smithi082 bash[77063]: cluster 2023-10-09T04:25:05.302639+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:06.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:06 smithi082 bash[79733]: cluster 2023-10-09T04:25:05.302639+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:07.339 INFO:teuthology.orchestra.run.smithi082.stdout:128849019104 2023-10-09T04:25:07.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:25:07.608 INFO:teuthology.orchestra.run.smithi082.stdout:399431958557 2023-10-09T04:25:07.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:25:07.623 INFO:teuthology.orchestra.run.smithi082.stdout:377957122082 2023-10-09T04:25:07.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:25:08.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:08 smithi153 bash[59565]: cluster 2023-10-09T04:25:07.303475+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:08.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:08 smithi082 bash[77063]: cluster 2023-10-09T04:25:07.303475+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:08.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:08 smithi082 bash[79733]: cluster 2023-10-09T04:25:07.303475+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:09 smithi153 bash[59565]: audit 2023-10-09T04:25:08.517382+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:09.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:09 smithi082 bash[77063]: audit 2023-10-09T04:25:08.517382+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:09.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:09 smithi082 bash[79733]: audit 2023-10-09T04:25:08.517382+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:09] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:25:09.989 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:09.989 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:09.990 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:10.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:10 smithi153 bash[59565]: cluster 2023-10-09T04:25:09.304333+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:10 smithi082 bash[77063]: cluster 2023-10-09T04:25:09.304333+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:10 smithi082 bash[79733]: cluster 2023-10-09T04:25:09.304333+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:11.004 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:11.005 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.021 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.022 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.025 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:12 smithi153 bash[59565]: cluster 2023-10-09T04:25:11.305280+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:12.760 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:12 smithi082 bash[77063]: cluster 2023-10-09T04:25:11.305280+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:12.760 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:12 smithi082 bash[79733]: cluster 2023-10-09T04:25:11.305280+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:14 smithi153 bash[59565]: cluster 2023-10-09T04:25:13.305690+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:14 smithi153 bash[59565]: audit 2023-10-09T04:25:14.106016+0000 mon.a (mon.0) 503 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:14 smithi153 bash[59565]: audit 2023-10-09T04:25:14.114552+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:14 smithi153 bash[59565]: audit 2023-10-09T04:25:14.252492+0000 mon.a (mon.0) 505 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:14.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[79733]: cluster 2023-10-09T04:25:13.305690+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:14.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[79733]: audit 2023-10-09T04:25:14.106016+0000 mon.a (mon.0) 503 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[79733]: audit 2023-10-09T04:25:14.114552+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[79733]: audit 2023-10-09T04:25:14.252492+0000 mon.a (mon.0) 505 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:14.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[77063]: cluster 2023-10-09T04:25:13.305690+0000 mgr.y (mgr.34107) 320 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:14.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[77063]: audit 2023-10-09T04:25:14.106016+0000 mon.a (mon.0) 503 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[77063]: audit 2023-10-09T04:25:14.114552+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:14.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:14 smithi082 bash[77063]: audit 2023-10-09T04:25:14.252492+0000 mon.a (mon.0) 505 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:15.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: audit 2023-10-09T04:25:14.808219+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: audit 2023-10-09T04:25:14.815653+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: audit 2023-10-09T04:25:15.284203+0000 mon.a (mon.0) 508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: audit 2023-10-09T04:25:15.285081+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: audit 2023-10-09T04:25:15.292017+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[77063]: cluster 2023-10-09T04:25:15.306195+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: audit 2023-10-09T04:25:14.808219+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: audit 2023-10-09T04:25:14.815653+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: audit 2023-10-09T04:25:15.284203+0000 mon.a (mon.0) 508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: audit 2023-10-09T04:25:15.285081+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:15.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: audit 2023-10-09T04:25:15.292017+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:15.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:15 smithi082 bash[79733]: cluster 2023-10-09T04:25:15.306195+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: audit 2023-10-09T04:25:14.808219+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: audit 2023-10-09T04:25:14.815653+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: audit 2023-10-09T04:25:15.284203+0000 mon.a (mon.0) 508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: audit 2023-10-09T04:25:15.285081+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: audit 2023-10-09T04:25:15.292017+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:25:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:15 smithi153 bash[59565]: cluster 2023-10-09T04:25:15.306195+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:18 smithi153 bash[59565]: cluster 2023-10-09T04:25:17.306938+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:18.714 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:18 smithi082 bash[77063]: cluster 2023-10-09T04:25:17.306938+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:18.715 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:18 smithi082 bash[79733]: cluster 2023-10-09T04:25:17.306938+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:19.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:19] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.33.4" 2023-10-09T04:25:20.444 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:20 smithi082 bash[77063]: cluster 2023-10-09T04:25:19.307630+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:20.444 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:20 smithi082 bash[79733]: cluster 2023-10-09T04:25:19.307630+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:20 smithi153 bash[59565]: cluster 2023-10-09T04:25:19.307630+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:21.321 INFO:teuthology.orchestra.run.smithi082.stdout:206158430410 2023-10-09T04:25:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:21 smithi082 bash[77063]: audit 2023-10-09T04:25:21.319776+0000 mon.a (mon.0) 511 : audit 0 from='client.? 172.21.15.82:0/2756516975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:25:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:21 smithi082 bash[79733]: audit 2023-10-09T04:25:21.319776+0000 mon.a (mon.0) 511 : audit 0 from='client.? 172.21.15.82:0/2756516975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:25:21.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:21 smithi153 bash[59565]: audit 2023-10-09T04:25:21.319776+0000 mon.a (mon.0) 511 : audit 0 from='client.? 172.21.15.82:0/2756516975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:25:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:22 smithi153 bash[59565]: cluster 2023-10-09T04:25:21.308419+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:22.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:22 smithi082 bash[77063]: cluster 2023-10-09T04:25:21.308419+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:22 smithi082 bash[79733]: cluster 2023-10-09T04:25:21.308419+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:23.512 INFO:teuthology.orchestra.run.smithi082.stdout:154618822875 2023-10-09T04:25:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:24 smithi153 bash[59565]: cluster 2023-10-09T04:25:23.308918+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:24 smithi153 bash[59565]: audit 2023-10-09T04:25:23.510936+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.82:0/4074419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:25:24.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:24 smithi082 bash[77063]: cluster 2023-10-09T04:25:23.308918+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:24.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:24 smithi082 bash[77063]: audit 2023-10-09T04:25:23.510936+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.82:0/4074419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:25:24.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:24 smithi082 bash[79733]: cluster 2023-10-09T04:25:23.308918+0000 mgr.y (mgr.34107) 325 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:24.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:24 smithi082 bash[79733]: audit 2023-10-09T04:25:23.510936+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.82:0/4074419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:25:24.785 INFO:teuthology.orchestra.run.smithi082.stdout:180388626644 2023-10-09T04:25:25.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:25 smithi153 bash[59565]: audit 2023-10-09T04:25:24.784153+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.82:0/1266361027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:25:25.680 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:25 smithi082 bash[77063]: audit 2023-10-09T04:25:24.784153+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.82:0/1266361027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:25:25.681 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:25 smithi082 bash[79733]: audit 2023-10-09T04:25:24.784153+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.82:0/1266361027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:25:25.681 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430407 got 206158430410 for osd.7 2023-10-09T04:25:25.682 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:25.971 INFO:teuthology.orchestra.run.smithi082.stdout:377957122086 2023-10-09T04:25:26.556 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822871 got 154618822875 for osd.5 2023-10-09T04:25:26.556 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:26 smithi153 bash[59565]: cluster 2023-10-09T04:25:25.310211+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:26 smithi153 bash[59565]: audit 2023-10-09T04:25:25.969907+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.82:0/2935097615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:25:26.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:26 smithi082 bash[77063]: cluster 2023-10-09T04:25:25.310211+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:26.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:26 smithi082 bash[77063]: audit 2023-10-09T04:25:25.969907+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.82:0/2935097615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:25:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:26 smithi082 bash[79733]: cluster 2023-10-09T04:25:25.310211+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:26 smithi082 bash[79733]: audit 2023-10-09T04:25:25.969907+0000 mon.a (mon.0) 512 : audit 0 from='client.? 172.21.15.82:0/2935097615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:25:27.533 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626640 got 180388626644 for osd.6 2023-10-09T04:25:27.534 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:27.978 INFO:teuthology.orchestra.run.smithi082.stdout:442381631507 2023-10-09T04:25:28.282 INFO:teuthology.orchestra.run.smithi082.stdout:128849019109 2023-10-09T04:25:28.400 INFO:teuthology.orchestra.run.smithi082.stdout:420906795036 2023-10-09T04:25:28.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[77063]: cluster 2023-10-09T04:25:27.310695+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:28.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[77063]: audit 2023-10-09T04:25:27.977653+0000 mon.a (mon.0) 513 : audit 0 from='client.? 172.21.15.82:0/639798084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:25:28.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[77063]: audit 2023-10-09T04:25:28.281136+0000 mon.a (mon.0) 514 : audit 0 from='client.? 172.21.15.82:0/2698794830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:25:28.625 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[79733]: cluster 2023-10-09T04:25:27.310695+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:28.625 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[79733]: audit 2023-10-09T04:25:27.977653+0000 mon.a (mon.0) 513 : audit 0 from='client.? 172.21.15.82:0/639798084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:25:28.625 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:28 smithi082 bash[79733]: audit 2023-10-09T04:25:28.281136+0000 mon.a (mon.0) 514 : audit 0 from='client.? 172.21.15.82:0/2698794830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:25:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:28 smithi153 bash[59565]: cluster 2023-10-09T04:25:27.310695+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:28 smithi153 bash[59565]: audit 2023-10-09T04:25:27.977653+0000 mon.a (mon.0) 513 : audit 0 from='client.? 172.21.15.82:0/639798084' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:25:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:28 smithi153 bash[59565]: audit 2023-10-09T04:25:28.281136+0000 mon.a (mon.0) 514 : audit 0 from='client.? 172.21.15.82:0/2698794830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:25:29.118 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122082 got 377957122086 for osd.0 2023-10-09T04:25:29.118 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:29.353 INFO:teuthology.orchestra.run.smithi082.stdout:399431958561 2023-10-09T04:25:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:29 smithi153 bash[59565]: audit 2023-10-09T04:25:28.399224+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.82:0/1968876074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:25:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:29 smithi153 bash[59565]: audit 2023-10-09T04:25:29.252709+0000 mon.a (mon.0) 515 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:29.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:29 smithi153 bash[59565]: audit 2023-10-09T04:25:29.351910+0000 mon.a (mon.0) 516 : audit 0 from='client.? 172.21.15.82:0/4047642883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:25:29.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[79733]: audit 2023-10-09T04:25:28.399224+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.82:0/1968876074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:25:29.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[79733]: audit 2023-10-09T04:25:29.252709+0000 mon.a (mon.0) 515 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:29.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[79733]: audit 2023-10-09T04:25:29.351910+0000 mon.a (mon.0) 516 : audit 0 from='client.? 172.21.15.82:0/4047642883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:25:29.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[77063]: audit 2023-10-09T04:25:28.399224+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.82:0/1968876074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:25:29.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[77063]: audit 2023-10-09T04:25:29.252709+0000 mon.a (mon.0) 515 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:29.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[77063]: audit 2023-10-09T04:25:29.351910+0000 mon.a (mon.0) 516 : audit 0 from='client.? 172.21.15.82:0/4047642883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:25:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:29] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T04:25:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:30 smithi153 bash[59565]: cluster 2023-10-09T04:25:29.311398+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:30 smithi082 bash[77063]: cluster 2023-10-09T04:25:29.311398+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:30 smithi082 bash[79733]: cluster 2023-10-09T04:25:29.311398+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:30.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631502 got 442381631507 for osd.3 2023-10-09T04:25:30.875 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:31.183 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795031 got 420906795036 for osd.2 2023-10-09T04:25:31.183 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:31.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019104 got 128849019109 for osd.4 2023-10-09T04:25:31.188 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:31.277 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958557 got 399431958561 for osd.1 2023-10-09T04:25:31.277 DEBUG:teuthology.parallel:result is None 2023-10-09T04:25:31.278 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:25:31.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:25:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:32 smithi153 bash[59565]: cluster 2023-10-09T04:25:31.312238+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:32.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:32 smithi082 bash[79733]: cluster 2023-10-09T04:25:31.312238+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:32.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:32 smithi082 bash[77063]: cluster 2023-10-09T04:25:31.312238+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:34 smithi153 bash[59565]: cluster 2023-10-09T04:25:33.312988+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:34.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:34 smithi082 bash[77063]: cluster 2023-10-09T04:25:33.312988+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:34.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:34 smithi082 bash[79733]: cluster 2023-10-09T04:25:33.312988+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:36.048 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:36 smithi153 bash[59565]: cluster 2023-10-09T04:25:35.313935+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:36.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:36 smithi082 bash[77063]: cluster 2023-10-09T04:25:35.313935+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:36.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:36 smithi082 bash[79733]: cluster 2023-10-09T04:25:35.313935+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:38.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:38 smithi153 bash[59565]: cluster 2023-10-09T04:25:37.314458+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:38.690 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:38 smithi082 bash[77063]: cluster 2023-10-09T04:25:37.314458+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:38.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:38 smithi082 bash[79733]: cluster 2023-10-09T04:25:37.314458+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:39.161 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:25:39.164 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:25:39.923 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-10-09T04:25:37.314197+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341268,"kb_used_data":22548,"kb_used_omap":34,"kb_used_meta":1318621,"kb_avail":748619948,"statfs":{"total":767960285184,"available":766586826752,"internally_reserved":0,"allocated":23089152,"data_stored":19011810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1350268629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004957"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802015+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:23:59.802015+0000","last_peered":"2023-10-09T04:23:59.802015+0000","last_clean":"2023-10-09T04:23:59.802015+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:23:59.802015+0000","last_undegraded":"2023-10-09T04:23:59.802015+0000","last_fullsized":"2023-10-09T04:23:59.802015+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804059+0000","last_change":"2023-10-09T04:23:18.765967+0000","last_active":"2023-10-09T04:24:00.804059+0000","last_peered":"2023-10-09T04:24:00.804059+0000","last_clean":"2023-10-09T04:24:00.804059+0000","last_became_active":"2023-10-09T04:23:18.765587+0000","last_became_peered":"2023-10-09T04:23:18.765587+0000","last_unstale":"2023-10-09T04:24:00.804059+0000","last_undegraded":"2023-10-09T04:24:00.804059+0000","last_fullsized":"2023-10-09T04:24:00.804059+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:10.976153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.815714+0000","last_change":"2023-10-09T04:24:00.815714+0000","last_active":"2023-10-09T04:24:00.815714+0000","last_peered":"2023-10-09T04:24:00.815714+0000","last_clean":"2023-10-09T04:24:00.815714+0000","last_became_active":"2023-10-09T04:24:00.815389+0000","last_became_peered":"2023-10-09T04:24:00.815389+0000","last_unstale":"2023-10-09T04:24:00.815714+0000","last_undegraded":"2023-10-09T04:24:00.815714+0000","last_fullsized":"2023-10-09T04:24:00.815714+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:39.857138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.819974+0000","last_change":"2023-10-09T04:24:00.819974+0000","last_active":"2023-10-09T04:24:00.819974+0000","last_peered":"2023-10-09T04:24:00.819974+0000","last_clean":"2023-10-09T04:24:00.819974+0000","last_became_active":"2023-10-09T04:24:00.819558+0000","last_became_peered":"2023-10-09T04:24:00.819558+0000","last_unstale":"2023-10-09T04:24:00.819974+0000","last_undegraded":"2023-10-09T04:24:00.819974+0000","last_fullsized":"2023-10-09T04:24:00.819974+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827513+0000","last_change":"2023-10-09T04:24:00.827513+0000","last_active":"2023-10-09T04:24:00.827513+0000","last_peered":"2023-10-09T04:24:00.827513+0000","last_clean":"2023-10-09T04:24:00.827513+0000","last_became_active":"2023-10-09T04:24:00.827037+0000","last_became_peered":"2023-10-09T04:24:00.827037+0000","last_unstale":"2023-10-09T04:24:00.827513+0000","last_undegraded":"2023-10-09T04:24:00.827513+0000","last_fullsized":"2023-10-09T04:24:00.827513+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794630+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:23:59.794630+0000","last_peered":"2023-10-09T04:23:59.794630+0000","last_clean":"2023-10-09T04:23:59.794630+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:23:59.794630+0000","last_undegraded":"2023-10-09T04:23:59.794630+0000","last_fullsized":"2023-10-09T04:23:59.794630+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827442+0000","last_change":"2023-10-09T04:24:00.827442+0000","last_active":"2023-10-09T04:24:00.827442+0000","last_peered":"2023-10-09T04:24:00.827442+0000","last_clean":"2023-10-09T04:24:00.827442+0000","last_became_active":"2023-10-09T04:24:00.827036+0000","last_became_peered":"2023-10-09T04:24:00.827036+0000","last_unstale":"2023-10-09T04:24:00.827442+0000","last_undegraded":"2023-10-09T04:24:00.827442+0000","last_fullsized":"2023-10-09T04:24:00.827442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794088+0000","last_change":"2023-10-09T04:22:50.701405+0000","last_active":"2023-10-09T04:23:59.794088+0000","last_peered":"2023-10-09T04:23:59.794088+0000","last_clean":"2023-10-09T04:23:59.794088+0000","last_became_active":"2023-10-09T04:22:50.700948+0000","last_became_peered":"2023-10-09T04:22:50.700948+0000","last_unstale":"2023-10-09T04:23:59.794088+0000","last_undegraded":"2023-10-09T04:23:59.794088+0000","last_fullsized":"2023-10-09T04:23:59.794088+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.811849+0000","last_change":"2023-10-09T04:24:00.811849+0000","last_active":"2023-10-09T04:24:00.811849+0000","last_peered":"2023-10-09T04:24:00.811849+0000","last_clean":"2023-10-09T04:24:00.811849+0000","last_became_active":"2023-10-09T04:24:00.811528+0000","last_became_peered":"2023-10-09T04:24:00.811528+0000","last_unstale":"2023-10-09T04:24:00.811849+0000","last_undegraded":"2023-10-09T04:24:00.811849+0000","last_fullsized":"2023-10-09T04:24:00.811849+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:25.981562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":42,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805109+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:24:00.805109+0000","last_peered":"2023-10-09T04:24:00.805109+0000","last_clean":"2023-10-09T04:24:00.805109+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:24:00.805109+0000","last_undegraded":"2023-10-09T04:24:00.805109+0000","last_fullsized":"2023-10-09T04:24:00.805109+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802722+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:23:59.802722+0000","last_peered":"2023-10-09T04:23:59.802722+0000","last_clean":"2023-10-09T04:23:59.802722+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:23:59.802722+0000","last_undegraded":"2023-10-09T04:23:59.802722+0000","last_fullsized":"2023-10-09T04:23:59.802722+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805271+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:24:00.805271+0000","last_peered":"2023-10-09T04:24:00.805271+0000","last_clean":"2023-10-09T04:24:00.805271+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:24:00.805271+0000","last_undegraded":"2023-10-09T04:24:00.805271+0000","last_fullsized":"2023-10-09T04:24:00.805271+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":151,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799287+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:23:59.799287+0000","last_peered":"2023-10-09T04:23:59.799287+0000","last_clean":"2023-10-09T04:23:59.799287+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:23:59.799287+0000","last_undegraded":"2023-10-09T04:23:59.799287+0000","last_fullsized":"2023-10-09T04:23:59.799287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822119+0000","last_change":"2023-10-09T04:24:00.822119+0000","last_active":"2023-10-09T04:24:00.822119+0000","last_peered":"2023-10-09T04:24:00.822119+0000","last_clean":"2023-10-09T04:24:00.822119+0000","last_became_active":"2023-10-09T04:24:00.821789+0000","last_became_peered":"2023-10-09T04:24:00.821789+0000","last_unstale":"2023-10-09T04:24:00.822119+0000","last_undegraded":"2023-10-09T04:24:00.822119+0000","last_fullsized":"2023-10-09T04:24:00.822119+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:09.080969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":324,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804070+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:24:00.804070+0000","last_peered":"2023-10-09T04:24:00.804070+0000","last_clean":"2023-10-09T04:24:00.804070+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:24:00.804070+0000","last_undegraded":"2023-10-09T04:24:00.804070+0000","last_fullsized":"2023-10-09T04:24:00.804070+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824546+0000","last_change":"2023-10-09T04:24:00.824546+0000","last_active":"2023-10-09T04:24:00.824546+0000","last_peered":"2023-10-09T04:24:00.824546+0000","last_clean":"2023-10-09T04:24:00.824546+0000","last_became_active":"2023-10-09T04:24:00.824005+0000","last_became_peered":"2023-10-09T04:24:00.824005+0000","last_unstale":"2023-10-09T04:24:00.824546+0000","last_undegraded":"2023-10-09T04:24:00.824546+0000","last_fullsized":"2023-10-09T04:24:00.824546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:46.407835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804761+0000","last_change":"2023-10-09T04:22:23.611501+0000","last_active":"2023-10-09T04:24:00.804761+0000","last_peered":"2023-10-09T04:24:00.804761+0000","last_clean":"2023-10-09T04:24:00.804761+0000","last_became_active":"2023-10-09T04:22:23.611116+0000","last_became_peered":"2023-10-09T04:22:23.611116+0000","last_unstale":"2023-10-09T04:24:00.804761+0000","last_undegraded":"2023-10-09T04:24:00.804761+0000","last_fullsized":"2023-10-09T04:24:00.804761+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:19.031531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806273+0000","last_change":"2023-10-09T04:23:18.773712+0000","last_active":"2023-10-09T04:24:00.806273+0000","last_peered":"2023-10-09T04:24:00.806273+0000","last_clean":"2023-10-09T04:24:00.806273+0000","last_became_active":"2023-10-09T04:23:18.773331+0000","last_became_peered":"2023-10-09T04:23:18.773331+0000","last_unstale":"2023-10-09T04:24:00.806273+0000","last_undegraded":"2023-10-09T04:24:00.806273+0000","last_fullsized":"2023-10-09T04:24:00.806273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:53.528302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":317,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797257+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:23:59.797257+0000","last_peered":"2023-10-09T04:23:59.797257+0000","last_clean":"2023-10-09T04:23:59.797257+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:23:59.797257+0000","last_undegraded":"2023-10-09T04:23:59.797257+0000","last_fullsized":"2023-10-09T04:23:59.797257+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805371+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:24:00.805371+0000","last_peered":"2023-10-09T04:24:00.805371+0000","last_clean":"2023-10-09T04:24:00.805371+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:24:00.805371+0000","last_undegraded":"2023-10-09T04:24:00.805371+0000","last_fullsized":"2023-10-09T04:24:00.805371+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823373+0000","last_change":"2023-10-09T04:24:00.823054+0000","last_active":"2023-10-09T04:24:00.823373+0000","last_peered":"2023-10-09T04:24:00.823373+0000","last_clean":"2023-10-09T04:24:00.823373+0000","last_became_active":"2023-10-09T04:24:00.822733+0000","last_became_peered":"2023-10-09T04:24:00.822733+0000","last_unstale":"2023-10-09T04:24:00.823373+0000","last_undegraded":"2023-10-09T04:24:00.823373+0000","last_fullsized":"2023-10-09T04:24:00.823373+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:13.369631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":648,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799128+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:23:59.799128+0000","last_peered":"2023-10-09T04:23:59.799128+0000","last_clean":"2023-10-09T04:23:59.799128+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:23:59.799128+0000","last_undegraded":"2023-10-09T04:23:59.799128+0000","last_fullsized":"2023-10-09T04:23:59.799128+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799465+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:23:59.799465+0000","last_peered":"2023-10-09T04:23:59.799465+0000","last_clean":"2023-10-09T04:23:59.799465+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:23:59.799465+0000","last_undegraded":"2023-10-09T04:23:59.799465+0000","last_fullsized":"2023-10-09T04:23:59.799465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.819863+0000","last_change":"2023-10-09T04:24:00.819863+0000","last_active":"2023-10-09T04:24:00.819863+0000","last_peered":"2023-10-09T04:24:00.819863+0000","last_clean":"2023-10-09T04:24:00.819863+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:24:00.819513+0000","last_unstale":"2023-10-09T04:24:00.819863+0000","last_undegraded":"2023-10-09T04:24:00.819863+0000","last_fullsized":"2023-10-09T04:24:00.819863+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:12.187156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798983+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:23:59.798983+0000","last_peered":"2023-10-09T04:23:59.798983+0000","last_clean":"2023-10-09T04:23:59.798983+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:23:59.798983+0000","last_undegraded":"2023-10-09T04:23:59.798983+0000","last_fullsized":"2023-10-09T04:23:59.798983+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":587,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792910+0000","last_change":"2023-10-09T04:23:18.767411+0000","last_active":"2023-10-09T04:23:59.792910+0000","last_peered":"2023-10-09T04:23:59.792910+0000","last_clean":"2023-10-09T04:23:59.792910+0000","last_became_active":"2023-10-09T04:23:18.766728+0000","last_became_peered":"2023-10-09T04:23:18.766728+0000","last_unstale":"2023-10-09T04:23:59.792910+0000","last_undegraded":"2023-10-09T04:23:59.792910+0000","last_fullsized":"2023-10-09T04:23:59.792910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":770,"num_read_kb":1698,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.803901+0000","last_change":"2023-10-09T04:22:50.696439+0000","last_active":"2023-10-09T04:24:00.803901+0000","last_peered":"2023-10-09T04:24:00.803901+0000","last_clean":"2023-10-09T04:24:00.803901+0000","last_became_active":"2023-10-09T04:22:50.696084+0000","last_became_peered":"2023-10-09T04:22:50.696084+0000","last_unstale":"2023-10-09T04:24:00.803901+0000","last_undegraded":"2023-10-09T04:24:00.803901+0000","last_fullsized":"2023-10-09T04:24:00.803901+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.931059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.807702+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:24:00.807702+0000","last_peered":"2023-10-09T04:24:00.807702+0000","last_clean":"2023-10-09T04:24:00.807702+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:24:00.807702+0000","last_undegraded":"2023-10-09T04:24:00.807702+0000","last_fullsized":"2023-10-09T04:24:00.807702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1209,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813355+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:24:00.813355+0000","last_peered":"2023-10-09T04:24:00.813355+0000","last_clean":"2023-10-09T04:24:00.813355+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:24:00.813355+0000","last_undegraded":"2023-10-09T04:24:00.813355+0000","last_fullsized":"2023-10-09T04:24:00.813355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814783+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:24:00.814783+0000","last_peered":"2023-10-09T04:24:00.814783+0000","last_clean":"2023-10-09T04:24:00.814783+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:24:00.814783+0000","last_undegraded":"2023-10-09T04:24:00.814783+0000","last_fullsized":"2023-10-09T04:24:00.814783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.801936+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:23:59.801936+0000","last_peered":"2023-10-09T04:23:59.801936+0000","last_clean":"2023-10-09T04:23:59.801936+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:23:59.801936+0000","last_undegraded":"2023-10-09T04:23:59.801936+0000","last_fullsized":"2023-10-09T04:23:59.801936+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":610,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792549+0000","last_change":"2023-10-09T04:22:50.705195+0000","last_active":"2023-10-09T04:23:59.792549+0000","last_peered":"2023-10-09T04:23:59.792549+0000","last_clean":"2023-10-09T04:23:59.792549+0000","last_became_active":"2023-10-09T04:22:50.704574+0000","last_became_peered":"2023-10-09T04:22:50.704574+0000","last_unstale":"2023-10-09T04:23:59.792549+0000","last_undegraded":"2023-10-09T04:23:59.792549+0000","last_fullsized":"2023-10-09T04:23:59.792549+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805058+0000","last_change":"2023-10-09T04:22:50.703397+0000","last_active":"2023-10-09T04:24:00.805058+0000","last_peered":"2023-10-09T04:24:00.805058+0000","last_clean":"2023-10-09T04:24:00.805058+0000","last_became_active":"2023-10-09T04:22:50.700260+0000","last_became_peered":"2023-10-09T04:22:50.700260+0000","last_unstale":"2023-10-09T04:24:00.805058+0000","last_undegraded":"2023-10-09T04:24:00.805058+0000","last_fullsized":"2023-10-09T04:24:00.805058+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:09.013765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813604+0000","last_change":"2023-10-09T04:24:00.813604+0000","last_active":"2023-10-09T04:24:00.813604+0000","last_peered":"2023-10-09T04:24:00.813604+0000","last_clean":"2023-10-09T04:24:00.813604+0000","last_became_active":"2023-10-09T04:24:00.813231+0000","last_became_peered":"2023-10-09T04:24:00.813231+0000","last_unstale":"2023-10-09T04:24:00.813604+0000","last_undegraded":"2023-10-09T04:24:00.813604+0000","last_fullsized":"2023-10-09T04:24:00.813604+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":473,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806172+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:24:00.806172+0000","last_peered":"2023-10-09T04:24:00.806172+0000","last_clean":"2023-10-09T04:24:00.806172+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:24:00.806172+0000","last_undegraded":"2023-10-09T04:24:00.806172+0000","last_fullsized":"2023-10-09T04:24:00.806172+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.825853+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:24:00.825853+0000","last_peered":"2023-10-09T04:24:00.825853+0000","last_clean":"2023-10-09T04:24:00.825853+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:24:00.825853+0000","last_undegraded":"2023-10-09T04:24:00.825853+0000","last_fullsized":"2023-10-09T04:24:00.825853+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822985+0000","last_change":"2023-10-09T04:24:00.822985+0000","last_active":"2023-10-09T04:24:00.822985+0000","last_peered":"2023-10-09T04:24:00.822985+0000","last_clean":"2023-10-09T04:24:00.822985+0000","last_became_active":"2023-10-09T04:24:00.822621+0000","last_became_peered":"2023-10-09T04:24:00.822621+0000","last_unstale":"2023-10-09T04:24:00.822985+0000","last_undegraded":"2023-10-09T04:24:00.822985+0000","last_fullsized":"2023-10-09T04:24:00.822985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:59.227822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802356+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:23:59.802356+0000","last_peered":"2023-10-09T04:23:59.802356+0000","last_clean":"2023-10-09T04:23:59.802356+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:23:59.802356+0000","last_undegraded":"2023-10-09T04:23:59.802356+0000","last_fullsized":"2023-10-09T04:23:59.802356+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802945+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:23:59.802945+0000","last_peered":"2023-10-09T04:23:59.802945+0000","last_clean":"2023-10-09T04:23:59.802945+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:23:59.802945+0000","last_undegraded":"2023-10-09T04:23:59.802945+0000","last_fullsized":"2023-10-09T04:23:59.802945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802914+0000","last_change":"2023-10-09T04:23:18.773293+0000","last_active":"2023-10-09T04:23:59.802914+0000","last_peered":"2023-10-09T04:23:59.802914+0000","last_clean":"2023-10-09T04:23:59.802914+0000","last_became_active":"2023-10-09T04:23:18.772947+0000","last_became_peered":"2023-10-09T04:23:18.772947+0000","last_unstale":"2023-10-09T04:23:59.802914+0000","last_undegraded":"2023-10-09T04:23:59.802914+0000","last_fullsized":"2023-10-09T04:23:59.802914+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799178+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:23:59.799178+0000","last_peered":"2023-10-09T04:23:59.799178+0000","last_clean":"2023-10-09T04:23:59.799178+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:23:59.799178+0000","last_undegraded":"2023-10-09T04:23:59.799178+0000","last_fullsized":"2023-10-09T04:23:59.799178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827521+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:24:00.827521+0000","last_peered":"2023-10-09T04:24:00.827521+0000","last_clean":"2023-10-09T04:24:00.827521+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:24:00.827521+0000","last_undegraded":"2023-10-09T04:24:00.827521+0000","last_fullsized":"2023-10-09T04:24:00.827521+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826876+0000","last_change":"2023-10-09T04:24:00.826876+0000","last_active":"2023-10-09T04:24:00.826876+0000","last_peered":"2023-10-09T04:24:00.826876+0000","last_clean":"2023-10-09T04:24:00.826876+0000","last_became_active":"2023-10-09T04:24:00.826469+0000","last_became_peered":"2023-10-09T04:24:00.826469+0000","last_unstale":"2023-10-09T04:24:00.826876+0000","last_undegraded":"2023-10-09T04:24:00.826876+0000","last_fullsized":"2023-10-09T04:24:00.826876+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:08.758269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804987+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:24:00.804987+0000","last_peered":"2023-10-09T04:24:00.804987+0000","last_clean":"2023-10-09T04:24:00.804987+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:24:00.804987+0000","last_undegraded":"2023-10-09T04:24:00.804987+0000","last_fullsized":"2023-10-09T04:24:00.804987+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822324+0000","last_change":"2023-10-09T04:24:00.822324+0000","last_active":"2023-10-09T04:24:00.822324+0000","last_peered":"2023-10-09T04:24:00.822324+0000","last_clean":"2023-10-09T04:24:00.822324+0000","last_became_active":"2023-10-09T04:24:00.821777+0000","last_became_peered":"2023-10-09T04:24:00.821777+0000","last_unstale":"2023-10-09T04:24:00.822324+0000","last_undegraded":"2023-10-09T04:24:00.822324+0000","last_fullsized":"2023-10-09T04:24:00.822324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798937+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:23:59.798937+0000","last_peered":"2023-10-09T04:23:59.798937+0000","last_clean":"2023-10-09T04:23:59.798937+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:23:59.798937+0000","last_undegraded":"2023-10-09T04:23:59.798937+0000","last_fullsized":"2023-10-09T04:23:59.798937+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797398+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:23:59.797398+0000","last_peered":"2023-10-09T04:23:59.797398+0000","last_clean":"2023-10-09T04:23:59.797398+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:23:59.797398+0000","last_undegraded":"2023-10-09T04:23:59.797398+0000","last_fullsized":"2023-10-09T04:23:59.797398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827553+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:24:00.827553+0000","last_peered":"2023-10-09T04:24:00.827553+0000","last_clean":"2023-10-09T04:24:00.827553+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:24:00.827553+0000","last_undegraded":"2023-10-09T04:24:00.827553+0000","last_fullsized":"2023-10-09T04:24:00.827553+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797418+0000","last_change":"2023-10-09T04:22:50.692658+0000","last_active":"2023-10-09T04:23:59.797418+0000","last_peered":"2023-10-09T04:23:59.797418+0000","last_clean":"2023-10-09T04:23:59.797418+0000","last_became_active":"2023-10-09T04:22:50.692027+0000","last_became_peered":"2023-10-09T04:22:50.692027+0000","last_unstale":"2023-10-09T04:23:59.797418+0000","last_undegraded":"2023-10-09T04:23:59.797418+0000","last_fullsized":"2023-10-09T04:23:59.797418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792610+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:23:59.792610+0000","last_peered":"2023-10-09T04:23:59.792610+0000","last_clean":"2023-10-09T04:23:59.792610+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:23:59.792610+0000","last_undegraded":"2023-10-09T04:23:59.792610+0000","last_fullsized":"2023-10-09T04:23:59.792610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797868+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:23:59.797868+0000","last_peered":"2023-10-09T04:23:59.797868+0000","last_clean":"2023-10-09T04:23:59.797868+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:23:59.797868+0000","last_undegraded":"2023-10-09T04:23:59.797868+0000","last_fullsized":"2023-10-09T04:23:59.797868+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822512+0000","last_change":"2023-10-09T04:24:00.822512+0000","last_active":"2023-10-09T04:24:00.822512+0000","last_peered":"2023-10-09T04:24:00.822512+0000","last_clean":"2023-10-09T04:24:00.822512+0000","last_became_active":"2023-10-09T04:24:00.822183+0000","last_became_peered":"2023-10-09T04:24:00.822183+0000","last_unstale":"2023-10-09T04:24:00.822512+0000","last_undegraded":"2023-10-09T04:24:00.822512+0000","last_fullsized":"2023-10-09T04:24:00.822512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.029015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799467+0000","last_change":"2023-10-09T04:22:23.612613+0000","last_active":"2023-10-09T04:23:59.799467+0000","last_peered":"2023-10-09T04:23:59.799467+0000","last_clean":"2023-10-09T04:23:59.799467+0000","last_became_active":"2023-10-09T04:22:23.611940+0000","last_became_peered":"2023-10-09T04:22:23.611940+0000","last_unstale":"2023-10-09T04:23:59.799467+0000","last_undegraded":"2023-10-09T04:23:59.799467+0000","last_fullsized":"2023-10-09T04:23:59.799467+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822959+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:24:00.822959+0000","last_peered":"2023-10-09T04:24:00.822959+0000","last_clean":"2023-10-09T04:24:00.822959+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:24:00.822959+0000","last_undegraded":"2023-10-09T04:24:00.822959+0000","last_fullsized":"2023-10-09T04:24:00.822959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.796146+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:23:59.796146+0000","last_peered":"2023-10-09T04:23:59.796146+0000","last_clean":"2023-10-09T04:23:59.796146+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:23:59.796146+0000","last_undegraded":"2023-10-09T04:23:59.796146+0000","last_fullsized":"2023-10-09T04:23:59.796146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822638+0000","last_change":"2023-10-09T04:24:00.822638+0000","last_active":"2023-10-09T04:24:00.822638+0000","last_peered":"2023-10-09T04:24:00.822638+0000","last_clean":"2023-10-09T04:24:00.822638+0000","last_became_active":"2023-10-09T04:24:00.822103+0000","last_became_peered":"2023-10-09T04:24:00.822103+0000","last_unstale":"2023-10-09T04:24:00.822638+0000","last_undegraded":"2023-10-09T04:24:00.822638+0000","last_fullsized":"2023-10-09T04:24:00.822638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826616+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:24:00.826616+0000","last_peered":"2023-10-09T04:24:00.826616+0000","last_clean":"2023-10-09T04:24:00.826616+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:24:00.826616+0000","last_undegraded":"2023-10-09T04:24:00.826616+0000","last_fullsized":"2023-10-09T04:24:00.826616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.821914+0000","last_change":"2023-10-09T04:24:00.821914+0000","last_active":"2023-10-09T04:24:00.821914+0000","last_peered":"2023-10-09T04:24:00.821914+0000","last_clean":"2023-10-09T04:24:00.821914+0000","last_became_active":"2023-10-09T04:24:00.821588+0000","last_became_peered":"2023-10-09T04:24:00.821588+0000","last_unstale":"2023-10-09T04:24:00.821914+0000","last_undegraded":"2023-10-09T04:24:00.821914+0000","last_fullsized":"2023-10-09T04:24:00.821914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:52.558769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797383+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:23:59.797383+0000","last_peered":"2023-10-09T04:23:59.797383+0000","last_clean":"2023-10-09T04:23:59.797383+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:23:59.797383+0000","last_undegraded":"2023-10-09T04:23:59.797383+0000","last_fullsized":"2023-10-09T04:23:59.797383+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802062+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:23:59.802062+0000","last_peered":"2023-10-09T04:23:59.802062+0000","last_clean":"2023-10-09T04:23:59.802062+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:23:59.802062+0000","last_undegraded":"2023-10-09T04:23:59.802062+0000","last_fullsized":"2023-10-09T04:23:59.802062+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802932+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:23:59.802932+0000","last_peered":"2023-10-09T04:23:59.802932+0000","last_clean":"2023-10-09T04:23:59.802932+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:23:59.802932+0000","last_undegraded":"2023-10-09T04:23:59.802932+0000","last_fullsized":"2023-10-09T04:23:59.802932+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805158+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:24:00.805158+0000","last_peered":"2023-10-09T04:24:00.805158+0000","last_clean":"2023-10-09T04:24:00.805158+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:24:00.805158+0000","last_undegraded":"2023-10-09T04:24:00.805158+0000","last_fullsized":"2023-10-09T04:24:00.805158+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824188+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:24:00.824188+0000","last_peered":"2023-10-09T04:24:00.824188+0000","last_clean":"2023-10-09T04:24:00.824188+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:24:00.824188+0000","last_undegraded":"2023-10-09T04:24:00.824188+0000","last_fullsized":"2023-10-09T04:24:00.824188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822436+0000","last_change":"2023-10-09T04:24:00.822436+0000","last_active":"2023-10-09T04:24:00.822436+0000","last_peered":"2023-10-09T04:24:00.822436+0000","last_clean":"2023-10-09T04:24:00.822436+0000","last_became_active":"2023-10-09T04:24:00.822108+0000","last_became_peered":"2023-10-09T04:24:00.822108+0000","last_unstale":"2023-10-09T04:24:00.822436+0000","last_undegraded":"2023-10-09T04:24:00.822436+0000","last_fullsized":"2023-10-09T04:24:00.822436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:24.737042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.793754+0000","last_change":"2023-10-09T04:22:50.698679+0000","last_active":"2023-10-09T04:23:59.793754+0000","last_peered":"2023-10-09T04:23:59.793754+0000","last_clean":"2023-10-09T04:23:59.793754+0000","last_became_active":"2023-10-09T04:22:50.698360+0000","last_became_peered":"2023-10-09T04:22:50.698360+0000","last_unstale":"2023-10-09T04:23:59.793754+0000","last_undegraded":"2023-10-09T04:23:59.793754+0000","last_fullsized":"2023-10-09T04:23:59.793754+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820037+0000","last_change":"2023-10-09T04:24:00.820037+0000","last_active":"2023-10-09T04:24:00.820037+0000","last_peered":"2023-10-09T04:24:00.820037+0000","last_clean":"2023-10-09T04:24:00.820037+0000","last_became_active":"2023-10-09T04:24:00.819650+0000","last_became_peered":"2023-10-09T04:24:00.819650+0000","last_unstale":"2023-10-09T04:24:00.820037+0000","last_undegraded":"2023-10-09T04:24:00.820037+0000","last_fullsized":"2023-10-09T04:24:00.820037+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:32.924825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.816756+0000","last_change":"2023-10-09T04:24:00.816214+0000","last_active":"2023-10-09T04:24:00.816756+0000","last_peered":"2023-10-09T04:24:00.816756+0000","last_clean":"2023-10-09T04:24:00.816756+0000","last_became_active":"2023-10-09T04:24:00.815867+0000","last_became_peered":"2023-10-09T04:24:00.815867+0000","last_unstale":"2023-10-09T04:24:00.816756+0000","last_undegraded":"2023-10-09T04:24:00.816756+0000","last_fullsized":"2023-10-09T04:24:00.816756+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:24.219649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799344+0000","last_change":"2023-10-09T04:22:50.702787+0000","last_active":"2023-10-09T04:23:59.799344+0000","last_peered":"2023-10-09T04:23:59.799344+0000","last_clean":"2023-10-09T04:23:59.799344+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:22:50.702392+0000","last_unstale":"2023-10-09T04:23:59.799344+0000","last_undegraded":"2023-10-09T04:23:59.799344+0000","last_fullsized":"2023-10-09T04:23:59.799344+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814141+0000","last_change":"2023-10-09T04:24:00.814141+0000","last_active":"2023-10-09T04:24:00.814141+0000","last_peered":"2023-10-09T04:24:00.814141+0000","last_clean":"2023-10-09T04:24:00.814141+0000","last_became_active":"2023-10-09T04:24:00.813799+0000","last_became_peered":"2023-10-09T04:24:00.813799+0000","last_unstale":"2023-10-09T04:24:00.814141+0000","last_undegraded":"2023-10-09T04:24:00.814141+0000","last_fullsized":"2023-10-09T04:24:00.814141+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:37.871798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822980+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:24:00.822980+0000","last_peered":"2023-10-09T04:24:00.822980+0000","last_clean":"2023-10-09T04:24:00.822980+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:24:00.822980+0000","last_undegraded":"2023-10-09T04:24:00.822980+0000","last_fullsized":"2023-10-09T04:24:00.822980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826025+0000","last_change":"2023-10-09T04:24:00.825647+0000","last_active":"2023-10-09T04:24:00.826025+0000","last_peered":"2023-10-09T04:24:00.826025+0000","last_clean":"2023-10-09T04:24:00.826025+0000","last_became_active":"2023-10-09T04:24:00.825310+0000","last_became_peered":"2023-10-09T04:24:00.825310+0000","last_unstale":"2023-10-09T04:24:00.826025+0000","last_undegraded":"2023-10-09T04:24:00.826025+0000","last_fullsized":"2023-10-09T04:24:00.826025+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:07.180568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826023+0000","last_change":"2023-10-09T04:24:00.826023+0000","last_active":"2023-10-09T04:24:00.826023+0000","last_peered":"2023-10-09T04:24:00.826023+0000","last_clean":"2023-10-09T04:24:00.826023+0000","last_became_active":"2023-10-09T04:24:00.825466+0000","last_became_peered":"2023-10-09T04:24:00.825466+0000","last_unstale":"2023-10-09T04:24:00.826023+0000","last_undegraded":"2023-10-09T04:24:00.826023+0000","last_fullsized":"2023-10-09T04:24:00.826023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:39.932991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.800944+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:23:59.800944+0000","last_peered":"2023-10-09T04:23:59.800944+0000","last_clean":"2023-10-09T04:23:59.800944+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:23:59.800944+0000","last_undegraded":"2023-10-09T04:23:59.800944+0000","last_fullsized":"2023-10-09T04:23:59.800944+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430414,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.56499999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92400000000000004,"15min":0.91600000000000004},"min":{"1min":0.624,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.212,"5min":1.375,"15min":1.375},"last":0.95699999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:24:55 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.92800000000000005,"15min":0.93000000000000005},"min":{"1min":0.68700000000000006,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.262,"5min":2.097,"15min":2.097},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.93999999999999995,"15min":0.93700000000000006},"min":{"1min":0.72499999999999998,"5min":0.503,"15min":0.503},"max":{"1min":1.2050000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.0980000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.69499999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1040000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.98399999999999999,"15min":0.98899999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.242,"15min":1.242},"last":1.0129999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.749,"5min":0.749,"15min":0.749},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.1220000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91000000000000003,"15min":0.91300000000000003},"min":{"1min":0.67100000000000004,"5min":0.55000000000000004,"15min":0.503},"max":{"1min":1.1399999999999999,"5min":1.4550000000000001,"15min":2.0419999999999998},"last":1.2150000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.70199999999999996,"5min":0.55200000000000005,"15min":0.495},"max":{"1min":1.2190000000000001,"5min":1.4399999999999999,"15min":2.1139999999999999},"last":0.98599999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.97099999999999997,"15min":0.98299999999999998},"min":{"1min":0.74199999999999999,"5min":0.51500000000000001,"15min":0.497},"max":{"1min":1.329,"5min":1.393,"15min":1.9510000000000001},"last":1.083},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0329999999999999,"15min":1.024},"min":{"1min":0.83099999999999996,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.3160000000000001,"5min":1.363,"15min":2.2349999999999999},"last":1.175}]},{"osd":6,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.046,"15min":1.0589999999999999},"min":{"1min":0.95399999999999996,"5min":0.51600000000000001,"15min":0.46100000000000002},"max":{"1min":1.5,"5min":1.5,"15min":2.3100000000000001},"last":1.038},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.044,"15min":1.044},"min":{"1min":0.77700000000000002,"5min":0.496,"15min":0.48199999999999998},"max":{"1min":1.248,"5min":1.4750000000000001,"15min":2.1909999999999998},"last":1.141}]}]},{"osd":6,"up_from":42,"seq":180388626647,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298728,"kb_used_data":1632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446424,"statfs":{"total":95995035648,"available":95689138176,"internally_reserved":0,"allocated":1671168,"data_stored":1171554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.88700000000000001,"15min":0.89100000000000001},"min":{"1min":0.57299999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2270000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.74099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:24:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1719999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.91400000000000003,"15min":0.90300000000000002},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.92100000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.95399999999999996,"15min":0.96499999999999997},"min":{"1min":0.64000000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.204,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.2250000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.0349999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.0800000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93700000000000006,"15min":0.91700000000000004},"min":{"1min":0.56799999999999995,"5min":0.51800000000000002,"15min":0.40899999999999997},"max":{"1min":1.147,"5min":1.367,"15min":1.663},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.99199999999999999,"15min":0.95799999999999996},"min":{"1min":0.65900000000000003,"5min":0.51200000000000001,"15min":0.443},"max":{"1min":1.319,"5min":1.405,"15min":1.548},"last":0.66400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0149999999999999,"15min":1.0369999999999999},"min":{"1min":0.67700000000000005,"5min":0.55900000000000005,"15min":0.50700000000000001},"max":{"1min":1.3009999999999999,"5min":1.498,"15min":3.9180000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0289999999999999,"15min":1.0049999999999999},"min":{"1min":0.745,"5min":0.56100000000000005,"15min":0.505},"max":{"1min":1.278,"5min":1.3320000000000001,"15min":1.635},"last":0.873}]},{"osd":7,"last update":"Mon Oct 9 04:25:13 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0860000000000001,"15min":1.0720000000000001},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.626},"max":{"1min":1.2609999999999999,"5min":1.5569999999999999,"15min":1.9570000000000001},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1679999999999999,"15min":1.159},"min":{"1min":0.85399999999999998,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3149999999999999,"5min":1.6279999999999999,"15min":1.9990000000000001},"last":0.97099999999999997}]}]},{"osd":1,"up_from":93,"seq":399431958563,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":6,"kb_used_meta":31545,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":585588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":32302092},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.67800000000000005,"15min":0.67500000000000004},"min":{"1min":0.443,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.95799999999999996,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.75,"5min":0.747,"15min":0.747},"min":{"1min":0.57099999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.101,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.77900000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:25:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.66400000000000003,"15min":0.67300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.76700000000000002,"15min":0.77900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.234,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.746}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.123,"5min":1.123,"15min":1.123},"last":0.69499999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92200000000000004,"15min":0.92100000000000004},"min":{"1min":0.75900000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.048,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.68200000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3480000000000001,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":0.90000000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.0149999999999999,"15min":1.018},"min":{"1min":0.81499999999999995,"5min":0.74099999999999999,"15min":0.74099999999999999},"max":{"1min":1.4570000000000001,"5min":1.819,"15min":1.819},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.98599999999999999,"15min":0.98699999999999999},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.2410000000000001,"5min":1.595,"15min":1.595},"last":1.032}]},{"osd":6,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.80900000000000005,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.397,"5min":1.877,"15min":1.877},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0229999999999999,"15min":1.0249999999999999},"min":{"1min":0.73599999999999999,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.2410000000000001,"5min":1.724,"15min":1.724},"last":0.97299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.014,"15min":1.008},"min":{"1min":0.70899999999999996,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.294,"5min":1.653,"15min":1.653},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.119,"15min":1.121},"min":{"1min":0.74299999999999999,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.421,"5min":1.786,"15min":1.786},"last":1.002}]}]},{"osd":0,"up_from":88,"seq":377957122089,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33304,"kb_used_data":1680,"kb_used_omap":9,"kb_used_meta":31606,"kb_avail":93711848,"statfs":{"total":95995035648,"available":95960932352,"internally_reserved":0,"allocated":1720320,"data_stored":1178304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":32365041},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.746},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.66500000000000004,"15min":0.67000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.79200000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.62},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.75700000000000001,"15min":0.77500000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.89500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":1.0569999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.93799999999999994}]},{"osd":4,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0249999999999999,"15min":1.0720000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1180000000000001,"5min":2.036,"15min":2.036},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.0209999999999999,"15min":1.0349999999999999},"min":{"1min":0.69499999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.25,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.0089999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.302,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.99299999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1379999999999999,"5min":1.613,"15min":1.613},"last":1.129}]},{"osd":6,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1,"15min":1.028},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2769999999999999,"5min":1.712,"15min":1.712},"last":1.032},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.012,"15min":1.012},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2869999999999999,"5min":1.641,"15min":1.641},"last":1.1060000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.093,"15min":1.123},"min":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2649999999999999,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":1.22},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.101,"15min":1.137},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1799999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":1.1639999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795038,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36728,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31540,"kb_avail":93708424,"statfs":{"total":95995035648,"available":95957426176,"internally_reserved":0,"allocated":5292032,"data_stored":4777245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":32297002},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.66700000000000004,"15min":0.69099999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.91000000000000003,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.86499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.72399999999999998,"15min":0.73599999999999999},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.70999999999999996,"15min":0.71199999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.90200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":2.2759999999999998,"5min":2.2759999999999998,"15min":2.2759999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.1899999999999999,"5min":2.1899999999999999,"15min":2.1899999999999999},"last":0.55500000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94299999999999995,"15min":0.94599999999999995},"min":{"1min":0.66100000000000003,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.083,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.94399999999999995,"15min":0.95799999999999996},"min":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1579999999999999,"5min":1.429,"15min":1.429},"last":0.82699999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.024},"min":{"1min":0.70099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2989999999999999,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.262,"5min":1.718,"15min":1.718},"last":0.92200000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.006,"15min":1.02},"min":{"1min":0.60399999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3440000000000001,"5min":1.855,"15min":1.855},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.05,"15min":1.0649999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2769999999999999,"5min":1.603,"15min":1.603},"last":1.0629999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.115,"15min":1.139},"min":{"1min":0.63900000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3480000000000001,"5min":1.825,"15min":1.825},"last":1.097},{"interface":"front","average":{"1min":0.995,"5min":1.077,"15min":1.091},"min":{"1min":0.67900000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2949999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.72199999999999998}]}]},{"osd":3,"up_from":103,"seq":442381631509,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32552,"kb_used_data":1056,"kb_used_omap":5,"kb_used_meta":31482,"kb_avail":93712600,"statfs":{"total":95995035648,"available":95961702400,"internally_reserved":0,"allocated":1081344,"data_stored":580896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":32237742},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":1.046}]},{"osd":1,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.93100000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.86099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":2.0470000000000002,"5min":2.0470000000000002,"15min":2.0470000000000002},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":1.1719999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.125},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.081}]},{"osd":6,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"min":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.99199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.145,"15min":1.145},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"min":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"max":{"1min":2.3980000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":1.099}]}]},{"osd":4,"up_from":30,"seq":128849019111,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84699999999999998,"15min":0.85699999999999998},"min":{"1min":0.54000000000000004,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.165,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0629999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.92900000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.90500000000000003,"15min":0.91000000000000003},"min":{"1min":0.59699999999999998,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1499999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90000000000000002,"15min":0.89400000000000002},"min":{"1min":0.747,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.2410000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.98499999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.629,"5min":0.502,"15min":0.502},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96099999999999997,"15min":0.95999999999999996},"min":{"1min":0.64100000000000001,"5min":0.623,"15min":0.623},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.1439999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.087,"15min":1.087},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.1990000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.96299999999999997,"15min":0.93200000000000005},"min":{"1min":0.65300000000000002,"5min":0.56399999999999995,"15min":0.433},"max":{"1min":1.3859999999999999,"5min":1.5269999999999999,"15min":1.5509999999999999},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.94599999999999995,"15min":0.89700000000000002},"min":{"1min":0.498,"5min":0.498,"15min":0.48399999999999999},"max":{"1min":1.2110000000000001,"5min":1.877,"15min":1.877},"last":0.78200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.99099999999999999,"15min":0.96299999999999997},"min":{"1min":0.755,"5min":0.56100000000000005,"15min":0.47099999999999997},"max":{"1min":1.389,"5min":1.389,"15min":1.7290000000000001},"last":0.749},{"interface":"front","average":{"1min":1.02,"5min":1.0549999999999999,"15min":1.026},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.46200000000000002},"max":{"1min":1.3740000000000001,"5min":1.583,"15min":1.796},"last":1.0800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1439999999999999,"15min":1.113},"min":{"1min":0.82599999999999996,"5min":0.68500000000000005,"15min":0.48699999999999999},"max":{"1min":1.3300000000000001,"5min":1.653,"15min":1.8600000000000001},"last":0.872},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1259999999999999,"15min":1.095},"min":{"1min":0.93300000000000005,"5min":0.76500000000000001,"15min":0.57699999999999996},"max":{"1min":1.4450000000000001,"5min":1.512,"15min":1.923},"last":1.0409999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822878,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302232,"kb_used_data":5136,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442920,"statfs":{"total":95995035648,"available":95685550080,"internally_reserved":0,"allocated":5259264,"data_stored":4770995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.87,"15min":0.872},"min":{"1min":0.63600000000000001,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.1399999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.74399999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.077}]},{"osd":1,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.99199999999999999,"15min":1.0009999999999999},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.268,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"min":{"1min":0.57699999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.21,"5min":1.645,"15min":1.645},"last":1.0409999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93100000000000005,"15min":0.93600000000000005},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0189999999999999,"15min":1.0209999999999999},"min":{"1min":0.79700000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.266,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.157}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.05},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.96699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:24:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.93700000000000006,"15min":0.93999999999999995},"min":{"1min":0.62,"5min":0.51600000000000001,"15min":0.47099999999999997},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":2.165},"last":0.995},{"interface":"front","average":{"1min":1.034,"5min":1.022,"15min":0.98499999999999999},"min":{"1min":0.59399999999999997,"5min":0.54600000000000004,"15min":0.47599999999999998},"max":{"1min":1.4059999999999999,"5min":3.7989999999999999,"15min":3.7989999999999999},"last":1.2390000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0229999999999999,"15min":1.0069999999999999},"min":{"1min":0.77000000000000002,"5min":0.50700000000000001,"15min":0.40899999999999997},"max":{"1min":1.5700000000000001,"5min":2.2999999999999998,"15min":2.2999999999999998},"last":1.117},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.083,"15min":1.0720000000000001},"min":{"1min":0.68500000000000005,"5min":0.58799999999999997,"15min":0.44800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":2.5840000000000001},"last":1.3089999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1100000000000001,"15min":1.1040000000000001},"min":{"1min":0.75,"5min":0.68899999999999995,"15min":0.42599999999999999},"max":{"1min":1.3540000000000001,"5min":1.6160000000000001,"15min":2.7050000000000001},"last":1.3520000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0820000000000001,"15min":1.0980000000000001},"min":{"1min":0.753,"5min":0.73999999999999999,"15min":0.47799999999999998},"max":{"1min":1.456,"5min":1.456,"15min":2.2829999999999999},"last":1.2230000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:25:39.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:25:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:39] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T04:25:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:40 smithi153 bash[59565]: audit 2023-10-09T04:25:39.146856+0000 mgr.y (mgr.34107) 333 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:40.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:40 smithi153 bash[59565]: cluster 2023-10-09T04:25:39.315240+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:40.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:40 smithi082 bash[77063]: audit 2023-10-09T04:25:39.146856+0000 mgr.y (mgr.34107) 333 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:40.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:40 smithi082 bash[77063]: cluster 2023-10-09T04:25:39.315240+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:40.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:40 smithi082 bash[79733]: audit 2023-10-09T04:25:39.146856+0000 mgr.y (mgr.34107) 333 : audit 0 from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:40.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:40 smithi082 bash[79733]: cluster 2023-10-09T04:25:39.315240+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:42 smithi153 bash[59565]: cluster 2023-10-09T04:25:41.316028+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:42 smithi082 bash[77063]: cluster 2023-10-09T04:25:41.316028+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:42 smithi082 bash[79733]: cluster 2023-10-09T04:25:41.316028+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:44 smithi153 bash[59565]: cluster 2023-10-09T04:25:43.316790+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:44 smithi153 bash[59565]: audit 2023-10-09T04:25:44.254120+0000 mon.a (mon.0) 517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:44 smithi082 bash[77063]: cluster 2023-10-09T04:25:43.316790+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:44 smithi082 bash[77063]: audit 2023-10-09T04:25:44.254120+0000 mon.a (mon.0) 517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:44 smithi082 bash[79733]: cluster 2023-10-09T04:25:43.316790+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:44 smithi082 bash[79733]: audit 2023-10-09T04:25:44.254120+0000 mon.a (mon.0) 517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:44.743 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:46 smithi153 bash[59565]: cluster 2023-10-09T04:25:45.317626+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:46 smithi082 bash[79733]: cluster 2023-10-09T04:25:45.317626+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:46 smithi082 bash[77063]: cluster 2023-10-09T04:25:45.317626+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:47.893 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:25:47.895 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:25:48.631 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":272,"stamp":"2023-10-09T04:25:47.317880+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341268,"kb_used_data":22548,"kb_used_omap":34,"kb_used_meta":1318621,"kb_avail":748619948,"statfs":{"total":767960285184,"available":766586826752,"internally_reserved":0,"allocated":23089152,"data_stored":19011810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1350268629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004624"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802015+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:23:59.802015+0000","last_peered":"2023-10-09T04:23:59.802015+0000","last_clean":"2023-10-09T04:23:59.802015+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:23:59.802015+0000","last_undegraded":"2023-10-09T04:23:59.802015+0000","last_fullsized":"2023-10-09T04:23:59.802015+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804059+0000","last_change":"2023-10-09T04:23:18.765967+0000","last_active":"2023-10-09T04:24:00.804059+0000","last_peered":"2023-10-09T04:24:00.804059+0000","last_clean":"2023-10-09T04:24:00.804059+0000","last_became_active":"2023-10-09T04:23:18.765587+0000","last_became_peered":"2023-10-09T04:23:18.765587+0000","last_unstale":"2023-10-09T04:24:00.804059+0000","last_undegraded":"2023-10-09T04:24:00.804059+0000","last_fullsized":"2023-10-09T04:24:00.804059+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:10.976153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.815714+0000","last_change":"2023-10-09T04:24:00.815714+0000","last_active":"2023-10-09T04:24:00.815714+0000","last_peered":"2023-10-09T04:24:00.815714+0000","last_clean":"2023-10-09T04:24:00.815714+0000","last_became_active":"2023-10-09T04:24:00.815389+0000","last_became_peered":"2023-10-09T04:24:00.815389+0000","last_unstale":"2023-10-09T04:24:00.815714+0000","last_undegraded":"2023-10-09T04:24:00.815714+0000","last_fullsized":"2023-10-09T04:24:00.815714+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:39.857138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.819974+0000","last_change":"2023-10-09T04:24:00.819974+0000","last_active":"2023-10-09T04:24:00.819974+0000","last_peered":"2023-10-09T04:24:00.819974+0000","last_clean":"2023-10-09T04:24:00.819974+0000","last_became_active":"2023-10-09T04:24:00.819558+0000","last_became_peered":"2023-10-09T04:24:00.819558+0000","last_unstale":"2023-10-09T04:24:00.819974+0000","last_undegraded":"2023-10-09T04:24:00.819974+0000","last_fullsized":"2023-10-09T04:24:00.819974+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827513+0000","last_change":"2023-10-09T04:24:00.827513+0000","last_active":"2023-10-09T04:24:00.827513+0000","last_peered":"2023-10-09T04:24:00.827513+0000","last_clean":"2023-10-09T04:24:00.827513+0000","last_became_active":"2023-10-09T04:24:00.827037+0000","last_became_peered":"2023-10-09T04:24:00.827037+0000","last_unstale":"2023-10-09T04:24:00.827513+0000","last_undegraded":"2023-10-09T04:24:00.827513+0000","last_fullsized":"2023-10-09T04:24:00.827513+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794630+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:23:59.794630+0000","last_peered":"2023-10-09T04:23:59.794630+0000","last_clean":"2023-10-09T04:23:59.794630+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:23:59.794630+0000","last_undegraded":"2023-10-09T04:23:59.794630+0000","last_fullsized":"2023-10-09T04:23:59.794630+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827442+0000","last_change":"2023-10-09T04:24:00.827442+0000","last_active":"2023-10-09T04:24:00.827442+0000","last_peered":"2023-10-09T04:24:00.827442+0000","last_clean":"2023-10-09T04:24:00.827442+0000","last_became_active":"2023-10-09T04:24:00.827036+0000","last_became_peered":"2023-10-09T04:24:00.827036+0000","last_unstale":"2023-10-09T04:24:00.827442+0000","last_undegraded":"2023-10-09T04:24:00.827442+0000","last_fullsized":"2023-10-09T04:24:00.827442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794088+0000","last_change":"2023-10-09T04:22:50.701405+0000","last_active":"2023-10-09T04:23:59.794088+0000","last_peered":"2023-10-09T04:23:59.794088+0000","last_clean":"2023-10-09T04:23:59.794088+0000","last_became_active":"2023-10-09T04:22:50.700948+0000","last_became_peered":"2023-10-09T04:22:50.700948+0000","last_unstale":"2023-10-09T04:23:59.794088+0000","last_undegraded":"2023-10-09T04:23:59.794088+0000","last_fullsized":"2023-10-09T04:23:59.794088+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813394+0000","last_change":"2023-10-09T04:24:00.811849+0000","last_active":"2023-10-09T04:24:00.813394+0000","last_peered":"2023-10-09T04:24:00.813394+0000","last_clean":"2023-10-09T04:24:00.813394+0000","last_became_active":"2023-10-09T04:24:00.811528+0000","last_became_peered":"2023-10-09T04:24:00.811528+0000","last_unstale":"2023-10-09T04:24:00.813394+0000","last_undegraded":"2023-10-09T04:24:00.813394+0000","last_fullsized":"2023-10-09T04:24:00.813394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:25.981562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":42,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805109+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:24:00.805109+0000","last_peered":"2023-10-09T04:24:00.805109+0000","last_clean":"2023-10-09T04:24:00.805109+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:24:00.805109+0000","last_undegraded":"2023-10-09T04:24:00.805109+0000","last_fullsized":"2023-10-09T04:24:00.805109+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802722+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:23:59.802722+0000","last_peered":"2023-10-09T04:23:59.802722+0000","last_clean":"2023-10-09T04:23:59.802722+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:23:59.802722+0000","last_undegraded":"2023-10-09T04:23:59.802722+0000","last_fullsized":"2023-10-09T04:23:59.802722+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805271+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:24:00.805271+0000","last_peered":"2023-10-09T04:24:00.805271+0000","last_clean":"2023-10-09T04:24:00.805271+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:24:00.805271+0000","last_undegraded":"2023-10-09T04:24:00.805271+0000","last_fullsized":"2023-10-09T04:24:00.805271+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":151,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799287+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:23:59.799287+0000","last_peered":"2023-10-09T04:23:59.799287+0000","last_clean":"2023-10-09T04:23:59.799287+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:23:59.799287+0000","last_undegraded":"2023-10-09T04:23:59.799287+0000","last_fullsized":"2023-10-09T04:23:59.799287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822119+0000","last_change":"2023-10-09T04:24:00.822119+0000","last_active":"2023-10-09T04:24:00.822119+0000","last_peered":"2023-10-09T04:24:00.822119+0000","last_clean":"2023-10-09T04:24:00.822119+0000","last_became_active":"2023-10-09T04:24:00.821789+0000","last_became_peered":"2023-10-09T04:24:00.821789+0000","last_unstale":"2023-10-09T04:24:00.822119+0000","last_undegraded":"2023-10-09T04:24:00.822119+0000","last_fullsized":"2023-10-09T04:24:00.822119+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:09.080969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":324,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804070+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:24:00.804070+0000","last_peered":"2023-10-09T04:24:00.804070+0000","last_clean":"2023-10-09T04:24:00.804070+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:24:00.804070+0000","last_undegraded":"2023-10-09T04:24:00.804070+0000","last_fullsized":"2023-10-09T04:24:00.804070+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824546+0000","last_change":"2023-10-09T04:24:00.824546+0000","last_active":"2023-10-09T04:24:00.824546+0000","last_peered":"2023-10-09T04:24:00.824546+0000","last_clean":"2023-10-09T04:24:00.824546+0000","last_became_active":"2023-10-09T04:24:00.824005+0000","last_became_peered":"2023-10-09T04:24:00.824005+0000","last_unstale":"2023-10-09T04:24:00.824546+0000","last_undegraded":"2023-10-09T04:24:00.824546+0000","last_fullsized":"2023-10-09T04:24:00.824546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:46.407835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804761+0000","last_change":"2023-10-09T04:22:23.611501+0000","last_active":"2023-10-09T04:24:00.804761+0000","last_peered":"2023-10-09T04:24:00.804761+0000","last_clean":"2023-10-09T04:24:00.804761+0000","last_became_active":"2023-10-09T04:22:23.611116+0000","last_became_peered":"2023-10-09T04:22:23.611116+0000","last_unstale":"2023-10-09T04:24:00.804761+0000","last_undegraded":"2023-10-09T04:24:00.804761+0000","last_fullsized":"2023-10-09T04:24:00.804761+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:19.031531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806273+0000","last_change":"2023-10-09T04:23:18.773712+0000","last_active":"2023-10-09T04:24:00.806273+0000","last_peered":"2023-10-09T04:24:00.806273+0000","last_clean":"2023-10-09T04:24:00.806273+0000","last_became_active":"2023-10-09T04:23:18.773331+0000","last_became_peered":"2023-10-09T04:23:18.773331+0000","last_unstale":"2023-10-09T04:24:00.806273+0000","last_undegraded":"2023-10-09T04:24:00.806273+0000","last_fullsized":"2023-10-09T04:24:00.806273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:53.528302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":317,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797257+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:23:59.797257+0000","last_peered":"2023-10-09T04:23:59.797257+0000","last_clean":"2023-10-09T04:23:59.797257+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:23:59.797257+0000","last_undegraded":"2023-10-09T04:23:59.797257+0000","last_fullsized":"2023-10-09T04:23:59.797257+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805371+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:24:00.805371+0000","last_peered":"2023-10-09T04:24:00.805371+0000","last_clean":"2023-10-09T04:24:00.805371+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:24:00.805371+0000","last_undegraded":"2023-10-09T04:24:00.805371+0000","last_fullsized":"2023-10-09T04:24:00.805371+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823373+0000","last_change":"2023-10-09T04:24:00.823054+0000","last_active":"2023-10-09T04:24:00.823373+0000","last_peered":"2023-10-09T04:24:00.823373+0000","last_clean":"2023-10-09T04:24:00.823373+0000","last_became_active":"2023-10-09T04:24:00.822733+0000","last_became_peered":"2023-10-09T04:24:00.822733+0000","last_unstale":"2023-10-09T04:24:00.823373+0000","last_undegraded":"2023-10-09T04:24:00.823373+0000","last_fullsized":"2023-10-09T04:24:00.823373+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:13.369631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":648,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799128+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:23:59.799128+0000","last_peered":"2023-10-09T04:23:59.799128+0000","last_clean":"2023-10-09T04:23:59.799128+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:23:59.799128+0000","last_undegraded":"2023-10-09T04:23:59.799128+0000","last_fullsized":"2023-10-09T04:23:59.799128+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799465+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:23:59.799465+0000","last_peered":"2023-10-09T04:23:59.799465+0000","last_clean":"2023-10-09T04:23:59.799465+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:23:59.799465+0000","last_undegraded":"2023-10-09T04:23:59.799465+0000","last_fullsized":"2023-10-09T04:23:59.799465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.819863+0000","last_change":"2023-10-09T04:24:00.819863+0000","last_active":"2023-10-09T04:24:00.819863+0000","last_peered":"2023-10-09T04:24:00.819863+0000","last_clean":"2023-10-09T04:24:00.819863+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:24:00.819513+0000","last_unstale":"2023-10-09T04:24:00.819863+0000","last_undegraded":"2023-10-09T04:24:00.819863+0000","last_fullsized":"2023-10-09T04:24:00.819863+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:12.187156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798983+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:23:59.798983+0000","last_peered":"2023-10-09T04:23:59.798983+0000","last_clean":"2023-10-09T04:23:59.798983+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:23:59.798983+0000","last_undegraded":"2023-10-09T04:23:59.798983+0000","last_fullsized":"2023-10-09T04:23:59.798983+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":587,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792910+0000","last_change":"2023-10-09T04:23:18.767411+0000","last_active":"2023-10-09T04:23:59.792910+0000","last_peered":"2023-10-09T04:23:59.792910+0000","last_clean":"2023-10-09T04:23:59.792910+0000","last_became_active":"2023-10-09T04:23:18.766728+0000","last_became_peered":"2023-10-09T04:23:18.766728+0000","last_unstale":"2023-10-09T04:23:59.792910+0000","last_undegraded":"2023-10-09T04:23:59.792910+0000","last_fullsized":"2023-10-09T04:23:59.792910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":770,"num_read_kb":1698,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.803901+0000","last_change":"2023-10-09T04:22:50.696439+0000","last_active":"2023-10-09T04:24:00.803901+0000","last_peered":"2023-10-09T04:24:00.803901+0000","last_clean":"2023-10-09T04:24:00.803901+0000","last_became_active":"2023-10-09T04:22:50.696084+0000","last_became_peered":"2023-10-09T04:22:50.696084+0000","last_unstale":"2023-10-09T04:24:00.803901+0000","last_undegraded":"2023-10-09T04:24:00.803901+0000","last_fullsized":"2023-10-09T04:24:00.803901+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.931059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.807702+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:24:00.807702+0000","last_peered":"2023-10-09T04:24:00.807702+0000","last_clean":"2023-10-09T04:24:00.807702+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:24:00.807702+0000","last_undegraded":"2023-10-09T04:24:00.807702+0000","last_fullsized":"2023-10-09T04:24:00.807702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1209,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813355+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:24:00.813355+0000","last_peered":"2023-10-09T04:24:00.813355+0000","last_clean":"2023-10-09T04:24:00.813355+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:24:00.813355+0000","last_undegraded":"2023-10-09T04:24:00.813355+0000","last_fullsized":"2023-10-09T04:24:00.813355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814783+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:24:00.814783+0000","last_peered":"2023-10-09T04:24:00.814783+0000","last_clean":"2023-10-09T04:24:00.814783+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:24:00.814783+0000","last_undegraded":"2023-10-09T04:24:00.814783+0000","last_fullsized":"2023-10-09T04:24:00.814783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.801936+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:23:59.801936+0000","last_peered":"2023-10-09T04:23:59.801936+0000","last_clean":"2023-10-09T04:23:59.801936+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:23:59.801936+0000","last_undegraded":"2023-10-09T04:23:59.801936+0000","last_fullsized":"2023-10-09T04:23:59.801936+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":610,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792549+0000","last_change":"2023-10-09T04:22:50.705195+0000","last_active":"2023-10-09T04:23:59.792549+0000","last_peered":"2023-10-09T04:23:59.792549+0000","last_clean":"2023-10-09T04:23:59.792549+0000","last_became_active":"2023-10-09T04:22:50.704574+0000","last_became_peered":"2023-10-09T04:22:50.704574+0000","last_unstale":"2023-10-09T04:23:59.792549+0000","last_undegraded":"2023-10-09T04:23:59.792549+0000","last_fullsized":"2023-10-09T04:23:59.792549+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805058+0000","last_change":"2023-10-09T04:22:50.703397+0000","last_active":"2023-10-09T04:24:00.805058+0000","last_peered":"2023-10-09T04:24:00.805058+0000","last_clean":"2023-10-09T04:24:00.805058+0000","last_became_active":"2023-10-09T04:22:50.700260+0000","last_became_peered":"2023-10-09T04:22:50.700260+0000","last_unstale":"2023-10-09T04:24:00.805058+0000","last_undegraded":"2023-10-09T04:24:00.805058+0000","last_fullsized":"2023-10-09T04:24:00.805058+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:09.013765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813604+0000","last_change":"2023-10-09T04:24:00.813604+0000","last_active":"2023-10-09T04:24:00.813604+0000","last_peered":"2023-10-09T04:24:00.813604+0000","last_clean":"2023-10-09T04:24:00.813604+0000","last_became_active":"2023-10-09T04:24:00.813231+0000","last_became_peered":"2023-10-09T04:24:00.813231+0000","last_unstale":"2023-10-09T04:24:00.813604+0000","last_undegraded":"2023-10-09T04:24:00.813604+0000","last_fullsized":"2023-10-09T04:24:00.813604+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":473,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806172+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:24:00.806172+0000","last_peered":"2023-10-09T04:24:00.806172+0000","last_clean":"2023-10-09T04:24:00.806172+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:24:00.806172+0000","last_undegraded":"2023-10-09T04:24:00.806172+0000","last_fullsized":"2023-10-09T04:24:00.806172+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.825853+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:24:00.825853+0000","last_peered":"2023-10-09T04:24:00.825853+0000","last_clean":"2023-10-09T04:24:00.825853+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:24:00.825853+0000","last_undegraded":"2023-10-09T04:24:00.825853+0000","last_fullsized":"2023-10-09T04:24:00.825853+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822985+0000","last_change":"2023-10-09T04:24:00.822985+0000","last_active":"2023-10-09T04:24:00.822985+0000","last_peered":"2023-10-09T04:24:00.822985+0000","last_clean":"2023-10-09T04:24:00.822985+0000","last_became_active":"2023-10-09T04:24:00.822621+0000","last_became_peered":"2023-10-09T04:24:00.822621+0000","last_unstale":"2023-10-09T04:24:00.822985+0000","last_undegraded":"2023-10-09T04:24:00.822985+0000","last_fullsized":"2023-10-09T04:24:00.822985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:59.227822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802356+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:23:59.802356+0000","last_peered":"2023-10-09T04:23:59.802356+0000","last_clean":"2023-10-09T04:23:59.802356+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:23:59.802356+0000","last_undegraded":"2023-10-09T04:23:59.802356+0000","last_fullsized":"2023-10-09T04:23:59.802356+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802945+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:23:59.802945+0000","last_peered":"2023-10-09T04:23:59.802945+0000","last_clean":"2023-10-09T04:23:59.802945+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:23:59.802945+0000","last_undegraded":"2023-10-09T04:23:59.802945+0000","last_fullsized":"2023-10-09T04:23:59.802945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802914+0000","last_change":"2023-10-09T04:23:18.773293+0000","last_active":"2023-10-09T04:23:59.802914+0000","last_peered":"2023-10-09T04:23:59.802914+0000","last_clean":"2023-10-09T04:23:59.802914+0000","last_became_active":"2023-10-09T04:23:18.772947+0000","last_became_peered":"2023-10-09T04:23:18.772947+0000","last_unstale":"2023-10-09T04:23:59.802914+0000","last_undegraded":"2023-10-09T04:23:59.802914+0000","last_fullsized":"2023-10-09T04:23:59.802914+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799178+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:23:59.799178+0000","last_peered":"2023-10-09T04:23:59.799178+0000","last_clean":"2023-10-09T04:23:59.799178+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:23:59.799178+0000","last_undegraded":"2023-10-09T04:23:59.799178+0000","last_fullsized":"2023-10-09T04:23:59.799178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827521+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:24:00.827521+0000","last_peered":"2023-10-09T04:24:00.827521+0000","last_clean":"2023-10-09T04:24:00.827521+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:24:00.827521+0000","last_undegraded":"2023-10-09T04:24:00.827521+0000","last_fullsized":"2023-10-09T04:24:00.827521+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826876+0000","last_change":"2023-10-09T04:24:00.826876+0000","last_active":"2023-10-09T04:24:00.826876+0000","last_peered":"2023-10-09T04:24:00.826876+0000","last_clean":"2023-10-09T04:24:00.826876+0000","last_became_active":"2023-10-09T04:24:00.826469+0000","last_became_peered":"2023-10-09T04:24:00.826469+0000","last_unstale":"2023-10-09T04:24:00.826876+0000","last_undegraded":"2023-10-09T04:24:00.826876+0000","last_fullsized":"2023-10-09T04:24:00.826876+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:08.758269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804987+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:24:00.804987+0000","last_peered":"2023-10-09T04:24:00.804987+0000","last_clean":"2023-10-09T04:24:00.804987+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:24:00.804987+0000","last_undegraded":"2023-10-09T04:24:00.804987+0000","last_fullsized":"2023-10-09T04:24:00.804987+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822324+0000","last_change":"2023-10-09T04:24:00.822324+0000","last_active":"2023-10-09T04:24:00.822324+0000","last_peered":"2023-10-09T04:24:00.822324+0000","last_clean":"2023-10-09T04:24:00.822324+0000","last_became_active":"2023-10-09T04:24:00.821777+0000","last_became_peered":"2023-10-09T04:24:00.821777+0000","last_unstale":"2023-10-09T04:24:00.822324+0000","last_undegraded":"2023-10-09T04:24:00.822324+0000","last_fullsized":"2023-10-09T04:24:00.822324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798937+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:23:59.798937+0000","last_peered":"2023-10-09T04:23:59.798937+0000","last_clean":"2023-10-09T04:23:59.798937+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:23:59.798937+0000","last_undegraded":"2023-10-09T04:23:59.798937+0000","last_fullsized":"2023-10-09T04:23:59.798937+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797398+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:23:59.797398+0000","last_peered":"2023-10-09T04:23:59.797398+0000","last_clean":"2023-10-09T04:23:59.797398+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:23:59.797398+0000","last_undegraded":"2023-10-09T04:23:59.797398+0000","last_fullsized":"2023-10-09T04:23:59.797398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827553+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:24:00.827553+0000","last_peered":"2023-10-09T04:24:00.827553+0000","last_clean":"2023-10-09T04:24:00.827553+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:24:00.827553+0000","last_undegraded":"2023-10-09T04:24:00.827553+0000","last_fullsized":"2023-10-09T04:24:00.827553+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797418+0000","last_change":"2023-10-09T04:22:50.692658+0000","last_active":"2023-10-09T04:23:59.797418+0000","last_peered":"2023-10-09T04:23:59.797418+0000","last_clean":"2023-10-09T04:23:59.797418+0000","last_became_active":"2023-10-09T04:22:50.692027+0000","last_became_peered":"2023-10-09T04:22:50.692027+0000","last_unstale":"2023-10-09T04:23:59.797418+0000","last_undegraded":"2023-10-09T04:23:59.797418+0000","last_fullsized":"2023-10-09T04:23:59.797418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792610+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:23:59.792610+0000","last_peered":"2023-10-09T04:23:59.792610+0000","last_clean":"2023-10-09T04:23:59.792610+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:23:59.792610+0000","last_undegraded":"2023-10-09T04:23:59.792610+0000","last_fullsized":"2023-10-09T04:23:59.792610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797868+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:23:59.797868+0000","last_peered":"2023-10-09T04:23:59.797868+0000","last_clean":"2023-10-09T04:23:59.797868+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:23:59.797868+0000","last_undegraded":"2023-10-09T04:23:59.797868+0000","last_fullsized":"2023-10-09T04:23:59.797868+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822512+0000","last_change":"2023-10-09T04:24:00.822512+0000","last_active":"2023-10-09T04:24:00.822512+0000","last_peered":"2023-10-09T04:24:00.822512+0000","last_clean":"2023-10-09T04:24:00.822512+0000","last_became_active":"2023-10-09T04:24:00.822183+0000","last_became_peered":"2023-10-09T04:24:00.822183+0000","last_unstale":"2023-10-09T04:24:00.822512+0000","last_undegraded":"2023-10-09T04:24:00.822512+0000","last_fullsized":"2023-10-09T04:24:00.822512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.029015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799467+0000","last_change":"2023-10-09T04:22:23.612613+0000","last_active":"2023-10-09T04:23:59.799467+0000","last_peered":"2023-10-09T04:23:59.799467+0000","last_clean":"2023-10-09T04:23:59.799467+0000","last_became_active":"2023-10-09T04:22:23.611940+0000","last_became_peered":"2023-10-09T04:22:23.611940+0000","last_unstale":"2023-10-09T04:23:59.799467+0000","last_undegraded":"2023-10-09T04:23:59.799467+0000","last_fullsized":"2023-10-09T04:23:59.799467+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822959+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:24:00.822959+0000","last_peered":"2023-10-09T04:24:00.822959+0000","last_clean":"2023-10-09T04:24:00.822959+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:24:00.822959+0000","last_undegraded":"2023-10-09T04:24:00.822959+0000","last_fullsized":"2023-10-09T04:24:00.822959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.796146+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:23:59.796146+0000","last_peered":"2023-10-09T04:23:59.796146+0000","last_clean":"2023-10-09T04:23:59.796146+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:23:59.796146+0000","last_undegraded":"2023-10-09T04:23:59.796146+0000","last_fullsized":"2023-10-09T04:23:59.796146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822638+0000","last_change":"2023-10-09T04:24:00.822638+0000","last_active":"2023-10-09T04:24:00.822638+0000","last_peered":"2023-10-09T04:24:00.822638+0000","last_clean":"2023-10-09T04:24:00.822638+0000","last_became_active":"2023-10-09T04:24:00.822103+0000","last_became_peered":"2023-10-09T04:24:00.822103+0000","last_unstale":"2023-10-09T04:24:00.822638+0000","last_undegraded":"2023-10-09T04:24:00.822638+0000","last_fullsized":"2023-10-09T04:24:00.822638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826616+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:24:00.826616+0000","last_peered":"2023-10-09T04:24:00.826616+0000","last_clean":"2023-10-09T04:24:00.826616+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:24:00.826616+0000","last_undegraded":"2023-10-09T04:24:00.826616+0000","last_fullsized":"2023-10-09T04:24:00.826616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.821914+0000","last_change":"2023-10-09T04:24:00.821914+0000","last_active":"2023-10-09T04:24:00.821914+0000","last_peered":"2023-10-09T04:24:00.821914+0000","last_clean":"2023-10-09T04:24:00.821914+0000","last_became_active":"2023-10-09T04:24:00.821588+0000","last_became_peered":"2023-10-09T04:24:00.821588+0000","last_unstale":"2023-10-09T04:24:00.821914+0000","last_undegraded":"2023-10-09T04:24:00.821914+0000","last_fullsized":"2023-10-09T04:24:00.821914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:52.558769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797383+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:23:59.797383+0000","last_peered":"2023-10-09T04:23:59.797383+0000","last_clean":"2023-10-09T04:23:59.797383+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:23:59.797383+0000","last_undegraded":"2023-10-09T04:23:59.797383+0000","last_fullsized":"2023-10-09T04:23:59.797383+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802062+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:23:59.802062+0000","last_peered":"2023-10-09T04:23:59.802062+0000","last_clean":"2023-10-09T04:23:59.802062+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:23:59.802062+0000","last_undegraded":"2023-10-09T04:23:59.802062+0000","last_fullsized":"2023-10-09T04:23:59.802062+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802932+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:23:59.802932+0000","last_peered":"2023-10-09T04:23:59.802932+0000","last_clean":"2023-10-09T04:23:59.802932+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:23:59.802932+0000","last_undegraded":"2023-10-09T04:23:59.802932+0000","last_fullsized":"2023-10-09T04:23:59.802932+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805158+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:24:00.805158+0000","last_peered":"2023-10-09T04:24:00.805158+0000","last_clean":"2023-10-09T04:24:00.805158+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:24:00.805158+0000","last_undegraded":"2023-10-09T04:24:00.805158+0000","last_fullsized":"2023-10-09T04:24:00.805158+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824188+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:24:00.824188+0000","last_peered":"2023-10-09T04:24:00.824188+0000","last_clean":"2023-10-09T04:24:00.824188+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:24:00.824188+0000","last_undegraded":"2023-10-09T04:24:00.824188+0000","last_fullsized":"2023-10-09T04:24:00.824188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822436+0000","last_change":"2023-10-09T04:24:00.822436+0000","last_active":"2023-10-09T04:24:00.822436+0000","last_peered":"2023-10-09T04:24:00.822436+0000","last_clean":"2023-10-09T04:24:00.822436+0000","last_became_active":"2023-10-09T04:24:00.822108+0000","last_became_peered":"2023-10-09T04:24:00.822108+0000","last_unstale":"2023-10-09T04:24:00.822436+0000","last_undegraded":"2023-10-09T04:24:00.822436+0000","last_fullsized":"2023-10-09T04:24:00.822436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:24.737042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.793754+0000","last_change":"2023-10-09T04:22:50.698679+0000","last_active":"2023-10-09T04:23:59.793754+0000","last_peered":"2023-10-09T04:23:59.793754+0000","last_clean":"2023-10-09T04:23:59.793754+0000","last_became_active":"2023-10-09T04:22:50.698360+0000","last_became_peered":"2023-10-09T04:22:50.698360+0000","last_unstale":"2023-10-09T04:23:59.793754+0000","last_undegraded":"2023-10-09T04:23:59.793754+0000","last_fullsized":"2023-10-09T04:23:59.793754+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820037+0000","last_change":"2023-10-09T04:24:00.820037+0000","last_active":"2023-10-09T04:24:00.820037+0000","last_peered":"2023-10-09T04:24:00.820037+0000","last_clean":"2023-10-09T04:24:00.820037+0000","last_became_active":"2023-10-09T04:24:00.819650+0000","last_became_peered":"2023-10-09T04:24:00.819650+0000","last_unstale":"2023-10-09T04:24:00.820037+0000","last_undegraded":"2023-10-09T04:24:00.820037+0000","last_fullsized":"2023-10-09T04:24:00.820037+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:32.924825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.816756+0000","last_change":"2023-10-09T04:24:00.816214+0000","last_active":"2023-10-09T04:24:00.816756+0000","last_peered":"2023-10-09T04:24:00.816756+0000","last_clean":"2023-10-09T04:24:00.816756+0000","last_became_active":"2023-10-09T04:24:00.815867+0000","last_became_peered":"2023-10-09T04:24:00.815867+0000","last_unstale":"2023-10-09T04:24:00.816756+0000","last_undegraded":"2023-10-09T04:24:00.816756+0000","last_fullsized":"2023-10-09T04:24:00.816756+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:24.219649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799344+0000","last_change":"2023-10-09T04:22:50.702787+0000","last_active":"2023-10-09T04:23:59.799344+0000","last_peered":"2023-10-09T04:23:59.799344+0000","last_clean":"2023-10-09T04:23:59.799344+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:22:50.702392+0000","last_unstale":"2023-10-09T04:23:59.799344+0000","last_undegraded":"2023-10-09T04:23:59.799344+0000","last_fullsized":"2023-10-09T04:23:59.799344+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814141+0000","last_change":"2023-10-09T04:24:00.814141+0000","last_active":"2023-10-09T04:24:00.814141+0000","last_peered":"2023-10-09T04:24:00.814141+0000","last_clean":"2023-10-09T04:24:00.814141+0000","last_became_active":"2023-10-09T04:24:00.813799+0000","last_became_peered":"2023-10-09T04:24:00.813799+0000","last_unstale":"2023-10-09T04:24:00.814141+0000","last_undegraded":"2023-10-09T04:24:00.814141+0000","last_fullsized":"2023-10-09T04:24:00.814141+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:37.871798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822980+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:24:00.822980+0000","last_peered":"2023-10-09T04:24:00.822980+0000","last_clean":"2023-10-09T04:24:00.822980+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:24:00.822980+0000","last_undegraded":"2023-10-09T04:24:00.822980+0000","last_fullsized":"2023-10-09T04:24:00.822980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826025+0000","last_change":"2023-10-09T04:24:00.825647+0000","last_active":"2023-10-09T04:24:00.826025+0000","last_peered":"2023-10-09T04:24:00.826025+0000","last_clean":"2023-10-09T04:24:00.826025+0000","last_became_active":"2023-10-09T04:24:00.825310+0000","last_became_peered":"2023-10-09T04:24:00.825310+0000","last_unstale":"2023-10-09T04:24:00.826025+0000","last_undegraded":"2023-10-09T04:24:00.826025+0000","last_fullsized":"2023-10-09T04:24:00.826025+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:07.180568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826023+0000","last_change":"2023-10-09T04:24:00.826023+0000","last_active":"2023-10-09T04:24:00.826023+0000","last_peered":"2023-10-09T04:24:00.826023+0000","last_clean":"2023-10-09T04:24:00.826023+0000","last_became_active":"2023-10-09T04:24:00.825466+0000","last_became_peered":"2023-10-09T04:24:00.825466+0000","last_unstale":"2023-10-09T04:24:00.826023+0000","last_undegraded":"2023-10-09T04:24:00.826023+0000","last_fullsized":"2023-10-09T04:24:00.826023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:39.932991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.800944+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:23:59.800944+0000","last_peered":"2023-10-09T04:23:59.800944+0000","last_clean":"2023-10-09T04:23:59.800944+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:23:59.800944+0000","last_undegraded":"2023-10-09T04:23:59.800944+0000","last_fullsized":"2023-10-09T04:23:59.800944+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430416,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.56499999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92400000000000004,"15min":0.91600000000000004},"min":{"1min":0.624,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.212,"5min":1.375,"15min":1.375},"last":1.1040000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:55 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.92800000000000005,"15min":0.93000000000000005},"min":{"1min":0.68700000000000006,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.262,"5min":2.097,"15min":2.097},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.93999999999999995,"15min":0.93700000000000006},"min":{"1min":0.72499999999999998,"5min":0.503,"15min":0.503},"max":{"1min":1.2050000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.93400000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.69499999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1040000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.98399999999999999,"15min":0.98899999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.242,"15min":1.242},"last":0.997}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.749,"5min":0.749,"15min":0.749},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.077}]},{"osd":4,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91000000000000003,"15min":0.91300000000000003},"min":{"1min":0.67100000000000004,"5min":0.55000000000000004,"15min":0.503},"max":{"1min":1.1399999999999999,"5min":1.4550000000000001,"15min":2.0419999999999998},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.70199999999999996,"5min":0.55200000000000005,"15min":0.495},"max":{"1min":1.2190000000000001,"5min":1.4399999999999999,"15min":2.1139999999999999},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.97099999999999997,"15min":0.98299999999999998},"min":{"1min":0.74199999999999999,"5min":0.51500000000000001,"15min":0.497},"max":{"1min":1.329,"5min":1.393,"15min":1.9510000000000001},"last":1.153},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0329999999999999,"15min":1.024},"min":{"1min":0.83099999999999996,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.3160000000000001,"5min":1.363,"15min":2.2349999999999999},"last":0.80900000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.046,"15min":1.0589999999999999},"min":{"1min":0.95399999999999996,"5min":0.51600000000000001,"15min":0.46100000000000002},"max":{"1min":1.5,"5min":1.5,"15min":2.3100000000000001},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.044,"15min":1.044},"min":{"1min":0.77700000000000002,"5min":0.496,"15min":0.48199999999999998},"max":{"1min":1.248,"5min":1.4750000000000001,"15min":2.1909999999999998},"last":1.0600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626649,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298728,"kb_used_data":1632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446424,"statfs":{"total":95995035648,"available":95689138176,"internally_reserved":0,"allocated":1671168,"data_stored":1171554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.88700000000000001,"15min":0.89100000000000001},"min":{"1min":0.57299999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2270000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.135}]},{"osd":1,"last update":"Mon Oct 9 04:24:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.89800000000000002,"15min":0.89600000000000002},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.1719999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.91400000000000003,"15min":0.90300000000000002},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":1.0920000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.95399999999999996,"15min":0.96499999999999997},"min":{"1min":0.64000000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.204,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.2250000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.1100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.1739999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93700000000000006,"15min":0.91700000000000004},"min":{"1min":0.56799999999999995,"5min":0.51800000000000002,"15min":0.40899999999999997},"max":{"1min":1.147,"5min":1.367,"15min":1.663},"last":0.99099999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.99199999999999999,"15min":0.95799999999999996},"min":{"1min":0.65900000000000003,"5min":0.51200000000000001,"15min":0.443},"max":{"1min":1.319,"5min":1.405,"15min":1.548},"last":1.034}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0149999999999999,"15min":1.0369999999999999},"min":{"1min":0.67700000000000005,"5min":0.55900000000000005,"15min":0.50700000000000001},"max":{"1min":1.3009999999999999,"5min":1.498,"15min":3.9180000000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0289999999999999,"15min":1.0049999999999999},"min":{"1min":0.745,"5min":0.56100000000000005,"15min":0.505},"max":{"1min":1.278,"5min":1.3320000000000001,"15min":1.635},"last":1.206}]},{"osd":7,"last update":"Mon Oct 9 04:25:13 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0860000000000001,"15min":1.0720000000000001},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.626},"max":{"1min":1.2609999999999999,"5min":1.5569999999999999,"15min":1.9570000000000001},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1679999999999999,"15min":1.159},"min":{"1min":0.85399999999999998,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3149999999999999,"5min":1.6279999999999999,"15min":1.9990000000000001},"last":1.2669999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958565,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":6,"kb_used_meta":31545,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":585588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":32302092},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.67800000000000005,"15min":0.67500000000000004},"min":{"1min":0.443,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.95799999999999996,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.75,"5min":0.747,"15min":0.747},"min":{"1min":0.57099999999999995,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.101,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.81599999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:25:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.66400000000000003,"15min":0.67300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.76700000000000002,"15min":0.77900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.234,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.0720000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":1.3240000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.123,"5min":1.123,"15min":1.123},"last":0.97299999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92200000000000004,"15min":0.92100000000000004},"min":{"1min":0.75900000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.68200000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3480000000000001,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.155}]},{"osd":5,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.0149999999999999,"15min":1.018},"min":{"1min":0.81499999999999995,"5min":0.74099999999999999,"15min":0.74099999999999999},"max":{"1min":1.4570000000000001,"5min":1.819,"15min":1.819},"last":1.4550000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.98599999999999999,"15min":0.98699999999999999},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.2410000000000001,"5min":1.595,"15min":1.595},"last":1.048}]},{"osd":6,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.80900000000000005,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.397,"5min":1.877,"15min":1.877},"last":1.5449999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0229999999999999,"15min":1.0249999999999999},"min":{"1min":0.73599999999999999,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.2410000000000001,"5min":1.724,"15min":1.724},"last":1.2769999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.014,"15min":1.008},"min":{"1min":0.70899999999999996,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.294,"5min":1.653,"15min":1.653},"last":1.349},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.119,"15min":1.121},"min":{"1min":0.74299999999999999,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.421,"5min":1.786,"15min":1.786},"last":1.4019999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122091,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33304,"kb_used_data":1680,"kb_used_omap":9,"kb_used_meta":31606,"kb_avail":93711848,"statfs":{"total":95995035648,"available":95960932352,"internally_reserved":0,"allocated":1720320,"data_stored":1178304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":32365041},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:24:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.66500000000000004,"15min":0.67000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.65000000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.75700000000000001,"15min":0.77500000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.76200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.876},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.59699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0249999999999999,"15min":1.0720000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1180000000000001,"5min":2.036,"15min":2.036},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.0209999999999999,"15min":1.0349999999999999},"min":{"1min":0.69499999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.25,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.93700000000000006}]},{"osd":5,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.302,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.99299999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1379999999999999,"5min":1.613,"15min":1.613},"last":1.0980000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1,"15min":1.028},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2769999999999999,"5min":1.712,"15min":1.712},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.012,"15min":1.012},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2869999999999999,"5min":1.641,"15min":1.641},"last":1.0229999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.093,"15min":1.123},"min":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2649999999999999,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":1.04},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.101,"15min":1.137},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1799999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.98199999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795040,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36728,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31540,"kb_avail":93708424,"statfs":{"total":95995035648,"available":95957426176,"internally_reserved":0,"allocated":5292032,"data_stored":4777245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":32297002},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.66700000000000004,"15min":0.69099999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.91000000000000003,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.72399999999999998,"15min":0.73599999999999999},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.70999999999999996,"15min":0.71199999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.68600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":2.2759999999999998,"5min":2.2759999999999998,"15min":2.2759999999999998},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.1899999999999999,"5min":2.1899999999999999,"15min":2.1899999999999999},"last":0.80900000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94299999999999995,"15min":0.94599999999999995},"min":{"1min":0.66100000000000003,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.083,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.94399999999999995,"15min":0.95799999999999996},"min":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1579999999999999,"5min":1.429,"15min":1.429},"last":0.95399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.024},"min":{"1min":0.70099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2989999999999999,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.874},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.262,"5min":1.718,"15min":1.718},"last":0.89000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.006,"15min":1.02},"min":{"1min":0.60399999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3440000000000001,"5min":1.855,"15min":1.855},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.05,"15min":1.0649999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2769999999999999,"5min":1.603,"15min":1.603},"last":1.012}]},{"osd":7,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.115,"15min":1.139},"min":{"1min":0.63900000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3480000000000001,"5min":1.825,"15min":1.825},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.995,"5min":1.077,"15min":1.091},"min":{"1min":0.67900000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2949999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.99299999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631511,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32552,"kb_used_data":1056,"kb_used_omap":5,"kb_used_meta":31482,"kb_avail":93712600,"statfs":{"total":95995035648,"available":95961702400,"internally_reserved":0,"allocated":1081344,"data_stored":580896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":32237742},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.627},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.50800000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":2.0470000000000002,"5min":2.0470000000000002,"15min":2.0470000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.65900000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.79200000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"min":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.88200000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.145,"15min":1.145},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"min":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"max":{"1min":2.3980000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":0.89700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019113,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84699999999999998,"15min":0.85699999999999998},"min":{"1min":0.54000000000000004,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.165,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0629999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.0620000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.90500000000000003,"15min":0.91000000000000003},"min":{"1min":0.59699999999999998,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1499999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90000000000000002,"15min":0.89400000000000002},"min":{"1min":0.747,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.2410000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":1.0269999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.629,"5min":0.502,"15min":0.502},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96099999999999997,"15min":0.95999999999999996},"min":{"1min":0.64100000000000001,"5min":0.623,"15min":0.623},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.98499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.087,"5min":1.087,"15min":1.087},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.23}]},{"osd":5,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.95499999999999996,"15min":0.93999999999999995},"min":{"1min":0.59999999999999998,"5min":0.56399999999999995,"15min":0.433},"max":{"1min":1.266,"5min":1.5269999999999999,"15min":1.5509999999999999},"last":1.105},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.93700000000000006,"15min":0.90500000000000003},"min":{"1min":0.69999999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.1970000000000001,"5min":1.877,"15min":1.877},"last":1.1539999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.99099999999999999,"15min":0.96299999999999997},"min":{"1min":0.755,"5min":0.56100000000000005,"15min":0.47099999999999997},"max":{"1min":1.389,"5min":1.389,"15min":1.7290000000000001},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.0549999999999999,"15min":1.026},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.46200000000000002},"max":{"1min":1.3740000000000001,"5min":1.583,"15min":1.796},"last":1.3280000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1439999999999999,"15min":1.113},"min":{"1min":0.82599999999999996,"5min":0.68500000000000005,"15min":0.48699999999999999},"max":{"1min":1.3300000000000001,"5min":1.653,"15min":1.8600000000000001},"last":1.427},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1259999999999999,"15min":1.095},"min":{"1min":0.93300000000000005,"5min":0.76500000000000001,"15min":0.57699999999999996},"max":{"1min":1.4450000000000001,"5min":1.512,"15min":1.923},"last":1.3819999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822880,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302232,"kb_used_data":5136,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442920,"statfs":{"total":95995035648,"available":95685550080,"internally_reserved":0,"allocated":5259264,"data_stored":4770995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.87,"15min":0.872},"min":{"1min":0.63600000000000001,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.1399999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":1.073},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.74399999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.82099999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:24:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.99199999999999999,"15min":1.0009999999999999},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.268,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"min":{"1min":0.57699999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.21,"5min":1.645,"15min":1.645},"last":1.095}]},{"osd":2,"last update":"Mon Oct 9 04:25:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93100000000000005,"15min":0.93600000000000005},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.042},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0189999999999999,"15min":1.0209999999999999},"min":{"1min":0.79700000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.266,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.96999999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.026},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.05},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.113}]},{"osd":4,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.93600000000000005,"15min":0.94699999999999995},"min":{"1min":0.70999999999999996,"5min":0.51600000000000001,"15min":0.47099999999999997},"max":{"1min":1.383,"5min":1.383,"15min":2.165},"last":0.85999999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.0429999999999999,"15min":0.997},"min":{"1min":0.72599999999999998,"5min":0.54600000000000004,"15min":0.47599999999999998},"max":{"1min":1.4319999999999999,"5min":3.7989999999999999,"15min":3.7989999999999999},"last":0.73999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0229999999999999,"15min":1.0069999999999999},"min":{"1min":0.77000000000000002,"5min":0.50700000000000001,"15min":0.40899999999999997},"max":{"1min":1.5700000000000001,"5min":2.2999999999999998,"15min":2.2999999999999998},"last":0.995},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.083,"15min":1.0720000000000001},"min":{"1min":0.68500000000000005,"5min":0.58799999999999997,"15min":0.44800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":2.5840000000000001},"last":0.93600000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1100000000000001,"15min":1.1040000000000001},"min":{"1min":0.75,"5min":0.68899999999999995,"15min":0.42599999999999999},"max":{"1min":1.3540000000000001,"5min":1.6160000000000001,"15min":2.7050000000000001},"last":1.153},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0820000000000001,"15min":1.0980000000000001},"min":{"1min":0.753,"5min":0.73999999999999999,"15min":0.47799999999999998},"max":{"1min":1.456,"5min":1.456,"15min":2.2829999999999999},"last":1.1359999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:25:48.636 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:25:48.636 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:25:48.636 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:25:48.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:25:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:48 smithi153 bash[59565]: cluster 2023-10-09T04:25:47.318330+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:48.688 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:48 smithi082 bash[79733]: cluster 2023-10-09T04:25:47.318330+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:48.688 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:48 smithi082 bash[77063]: cluster 2023-10-09T04:25:47.318330+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:49 smithi153 bash[59565]: audit 2023-10-09T04:25:47.874818+0000 mgr.y (mgr.34107) 339 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:49.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:49 smithi082 bash[77063]: audit 2023-10-09T04:25:47.874818+0000 mgr.y (mgr.34107) 339 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:49.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:49 smithi082 bash[79733]: audit 2023-10-09T04:25:47.874818+0000 mgr.y (mgr.34107) 339 : audit 0 from='client.44627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:49] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:25:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:50 smithi153 bash[59565]: cluster 2023-10-09T04:25:49.319024+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:50.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:50 smithi082 bash[77063]: cluster 2023-10-09T04:25:49.319024+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:50.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:50 smithi082 bash[79733]: cluster 2023-10-09T04:25:49.319024+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:52 smithi153 bash[59565]: cluster 2023-10-09T04:25:51.319898+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:52 smithi082 bash[77063]: cluster 2023-10-09T04:25:51.319898+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:52.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:52 smithi082 bash[79733]: cluster 2023-10-09T04:25:51.319898+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:53.455 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:54.617 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:54 smithi082 bash[77063]: cluster 2023-10-09T04:25:53.320626+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:54.617 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:54 smithi082 bash[79733]: cluster 2023-10-09T04:25:53.320626+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:54 smithi153 bash[59565]: cluster 2023-10-09T04:25:53.320626+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:56.533 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:25:56.533 INFO:teuthology.orchestra.run.smithi082.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:25:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:56 smithi153 bash[59565]: cluster 2023-10-09T04:25:55.321731+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:56.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:56 smithi082 bash[77063]: cluster 2023-10-09T04:25:55.321731+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:56.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:56 smithi082 bash[79733]: cluster 2023-10-09T04:25:55.321731+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:57.254 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:25:57.254 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:25:57.263 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T04:25:57.263 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:25:57.276 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:25:57.276 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:25:57.276 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T04:25:57.278 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:25:57.278 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T04:25:57.278 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:25:57.278 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T04:25:57.278 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:25:57.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:25:57.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:57 smithi153 bash[59565]: audit 2023-10-09T04:25:56.531737+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.82:0/1020338462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:25:57.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:57 smithi082 bash[77063]: audit 2023-10-09T04:25:56.531737+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.82:0/1020338462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:25:57.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:57 smithi082 bash[79733]: audit 2023-10-09T04:25:56.531737+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.82:0/1020338462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:25:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:58 smithi153 bash[59565]: cluster 2023-10-09T04:25:57.322398+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:58 smithi082 bash[77063]: cluster 2023-10-09T04:25:57.322398+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:58.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:58 smithi082 bash[79733]: cluster 2023-10-09T04:25:57.322398+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:25:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:25:59 smithi153 bash[59565]: audit 2023-10-09T04:25:59.254325+0000 mon.a (mon.0) 518 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:59.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:25:59 smithi082 bash[79733]: audit 2023-10-09T04:25:59.254325+0000 mon.a (mon.0) 518 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:59.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:25:59 smithi082 bash[77063]: audit 2023-10-09T04:25:59.254325+0000 mon.a (mon.0) 518 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:25:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:25:59] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:26:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:00 smithi082 bash[77063]: cluster 2023-10-09T04:25:59.323220+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:00 smithi082 bash[79733]: cluster 2023-10-09T04:25:59.323220+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:00.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:00 smithi153 bash[59565]: cluster 2023-10-09T04:25:59.323220+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:02.075 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:02.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:02 smithi082 bash[77063]: cluster 2023-10-09T04:26:01.324119+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:02.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:02 smithi082 bash[79733]: cluster 2023-10-09T04:26:01.324119+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:02 smithi153 bash[59565]: cluster 2023-10-09T04:26:01.324119+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:04.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:04 smithi082 bash[77063]: cluster 2023-10-09T04:26:03.324861+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:04.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:04 smithi082 bash[79733]: cluster 2023-10-09T04:26:03.324861+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:04.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:04 smithi153 bash[59565]: cluster 2023-10-09T04:26:03.324861+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:05.108 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:26:05.110 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:26:05.926 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2023-10-09T04:26:03.324416+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341268,"kb_used_data":22548,"kb_used_omap":34,"kb_used_meta":1318621,"kb_avail":748619948,"statfs":{"total":767960285184,"available":766586826752,"internally_reserved":0,"allocated":23089152,"data_stored":19011810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1350268629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005113"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802015+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:23:59.802015+0000","last_peered":"2023-10-09T04:23:59.802015+0000","last_clean":"2023-10-09T04:23:59.802015+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:23:59.802015+0000","last_undegraded":"2023-10-09T04:23:59.802015+0000","last_fullsized":"2023-10-09T04:23:59.802015+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804059+0000","last_change":"2023-10-09T04:23:18.765967+0000","last_active":"2023-10-09T04:24:00.804059+0000","last_peered":"2023-10-09T04:24:00.804059+0000","last_clean":"2023-10-09T04:24:00.804059+0000","last_became_active":"2023-10-09T04:23:18.765587+0000","last_became_peered":"2023-10-09T04:23:18.765587+0000","last_unstale":"2023-10-09T04:24:00.804059+0000","last_undegraded":"2023-10-09T04:24:00.804059+0000","last_fullsized":"2023-10-09T04:24:00.804059+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:10.976153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.815714+0000","last_change":"2023-10-09T04:24:00.815714+0000","last_active":"2023-10-09T04:24:00.815714+0000","last_peered":"2023-10-09T04:24:00.815714+0000","last_clean":"2023-10-09T04:24:00.815714+0000","last_became_active":"2023-10-09T04:24:00.815389+0000","last_became_peered":"2023-10-09T04:24:00.815389+0000","last_unstale":"2023-10-09T04:24:00.815714+0000","last_undegraded":"2023-10-09T04:24:00.815714+0000","last_fullsized":"2023-10-09T04:24:00.815714+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:39.857138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":80,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820442+0000","last_change":"2023-10-09T04:24:00.819974+0000","last_active":"2023-10-09T04:24:00.820442+0000","last_peered":"2023-10-09T04:24:00.820442+0000","last_clean":"2023-10-09T04:24:00.820442+0000","last_became_active":"2023-10-09T04:24:00.819558+0000","last_became_peered":"2023-10-09T04:24:00.819558+0000","last_unstale":"2023-10-09T04:24:00.820442+0000","last_undegraded":"2023-10-09T04:24:00.820442+0000","last_fullsized":"2023-10-09T04:24:00.820442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827513+0000","last_change":"2023-10-09T04:24:00.827513+0000","last_active":"2023-10-09T04:24:00.827513+0000","last_peered":"2023-10-09T04:24:00.827513+0000","last_clean":"2023-10-09T04:24:00.827513+0000","last_became_active":"2023-10-09T04:24:00.827037+0000","last_became_peered":"2023-10-09T04:24:00.827037+0000","last_unstale":"2023-10-09T04:24:00.827513+0000","last_undegraded":"2023-10-09T04:24:00.827513+0000","last_fullsized":"2023-10-09T04:24:00.827513+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794630+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:23:59.794630+0000","last_peered":"2023-10-09T04:23:59.794630+0000","last_clean":"2023-10-09T04:23:59.794630+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:23:59.794630+0000","last_undegraded":"2023-10-09T04:23:59.794630+0000","last_fullsized":"2023-10-09T04:23:59.794630+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827442+0000","last_change":"2023-10-09T04:24:00.827442+0000","last_active":"2023-10-09T04:24:00.827442+0000","last_peered":"2023-10-09T04:24:00.827442+0000","last_clean":"2023-10-09T04:24:00.827442+0000","last_became_active":"2023-10-09T04:24:00.827036+0000","last_became_peered":"2023-10-09T04:24:00.827036+0000","last_unstale":"2023-10-09T04:24:00.827442+0000","last_undegraded":"2023-10-09T04:24:00.827442+0000","last_fullsized":"2023-10-09T04:24:00.827442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794088+0000","last_change":"2023-10-09T04:22:50.701405+0000","last_active":"2023-10-09T04:23:59.794088+0000","last_peered":"2023-10-09T04:23:59.794088+0000","last_clean":"2023-10-09T04:23:59.794088+0000","last_became_active":"2023-10-09T04:22:50.700948+0000","last_became_peered":"2023-10-09T04:22:50.700948+0000","last_unstale":"2023-10-09T04:23:59.794088+0000","last_undegraded":"2023-10-09T04:23:59.794088+0000","last_fullsized":"2023-10-09T04:23:59.794088+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813394+0000","last_change":"2023-10-09T04:24:00.811849+0000","last_active":"2023-10-09T04:24:00.813394+0000","last_peered":"2023-10-09T04:24:00.813394+0000","last_clean":"2023-10-09T04:24:00.813394+0000","last_became_active":"2023-10-09T04:24:00.811528+0000","last_became_peered":"2023-10-09T04:24:00.811528+0000","last_unstale":"2023-10-09T04:24:00.813394+0000","last_undegraded":"2023-10-09T04:24:00.813394+0000","last_fullsized":"2023-10-09T04:24:00.813394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:25.981562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":42,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805109+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:24:00.805109+0000","last_peered":"2023-10-09T04:24:00.805109+0000","last_clean":"2023-10-09T04:24:00.805109+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:24:00.805109+0000","last_undegraded":"2023-10-09T04:24:00.805109+0000","last_fullsized":"2023-10-09T04:24:00.805109+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802722+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:23:59.802722+0000","last_peered":"2023-10-09T04:23:59.802722+0000","last_clean":"2023-10-09T04:23:59.802722+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:23:59.802722+0000","last_undegraded":"2023-10-09T04:23:59.802722+0000","last_fullsized":"2023-10-09T04:23:59.802722+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805271+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:24:00.805271+0000","last_peered":"2023-10-09T04:24:00.805271+0000","last_clean":"2023-10-09T04:24:00.805271+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:24:00.805271+0000","last_undegraded":"2023-10-09T04:24:00.805271+0000","last_fullsized":"2023-10-09T04:24:00.805271+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":151,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799287+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:23:59.799287+0000","last_peered":"2023-10-09T04:23:59.799287+0000","last_clean":"2023-10-09T04:23:59.799287+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:23:59.799287+0000","last_undegraded":"2023-10-09T04:23:59.799287+0000","last_fullsized":"2023-10-09T04:23:59.799287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823561+0000","last_change":"2023-10-09T04:24:00.822119+0000","last_active":"2023-10-09T04:24:00.823561+0000","last_peered":"2023-10-09T04:24:00.823561+0000","last_clean":"2023-10-09T04:24:00.823561+0000","last_became_active":"2023-10-09T04:24:00.821789+0000","last_became_peered":"2023-10-09T04:24:00.821789+0000","last_unstale":"2023-10-09T04:24:00.823561+0000","last_undegraded":"2023-10-09T04:24:00.823561+0000","last_fullsized":"2023-10-09T04:24:00.823561+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:09.080969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":324,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804070+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:24:00.804070+0000","last_peered":"2023-10-09T04:24:00.804070+0000","last_clean":"2023-10-09T04:24:00.804070+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:24:00.804070+0000","last_undegraded":"2023-10-09T04:24:00.804070+0000","last_fullsized":"2023-10-09T04:24:00.804070+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824546+0000","last_change":"2023-10-09T04:24:00.824546+0000","last_active":"2023-10-09T04:24:00.824546+0000","last_peered":"2023-10-09T04:24:00.824546+0000","last_clean":"2023-10-09T04:24:00.824546+0000","last_became_active":"2023-10-09T04:24:00.824005+0000","last_became_peered":"2023-10-09T04:24:00.824005+0000","last_unstale":"2023-10-09T04:24:00.824546+0000","last_undegraded":"2023-10-09T04:24:00.824546+0000","last_fullsized":"2023-10-09T04:24:00.824546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:46.407835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804761+0000","last_change":"2023-10-09T04:22:23.611501+0000","last_active":"2023-10-09T04:24:00.804761+0000","last_peered":"2023-10-09T04:24:00.804761+0000","last_clean":"2023-10-09T04:24:00.804761+0000","last_became_active":"2023-10-09T04:22:23.611116+0000","last_became_peered":"2023-10-09T04:22:23.611116+0000","last_unstale":"2023-10-09T04:24:00.804761+0000","last_undegraded":"2023-10-09T04:24:00.804761+0000","last_fullsized":"2023-10-09T04:24:00.804761+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:19.031531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806273+0000","last_change":"2023-10-09T04:23:18.773712+0000","last_active":"2023-10-09T04:24:00.806273+0000","last_peered":"2023-10-09T04:24:00.806273+0000","last_clean":"2023-10-09T04:24:00.806273+0000","last_became_active":"2023-10-09T04:23:18.773331+0000","last_became_peered":"2023-10-09T04:23:18.773331+0000","last_unstale":"2023-10-09T04:24:00.806273+0000","last_undegraded":"2023-10-09T04:24:00.806273+0000","last_fullsized":"2023-10-09T04:24:00.806273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:53.528302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":317,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797257+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:23:59.797257+0000","last_peered":"2023-10-09T04:23:59.797257+0000","last_clean":"2023-10-09T04:23:59.797257+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:23:59.797257+0000","last_undegraded":"2023-10-09T04:23:59.797257+0000","last_fullsized":"2023-10-09T04:23:59.797257+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805371+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:24:00.805371+0000","last_peered":"2023-10-09T04:24:00.805371+0000","last_clean":"2023-10-09T04:24:00.805371+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:24:00.805371+0000","last_undegraded":"2023-10-09T04:24:00.805371+0000","last_fullsized":"2023-10-09T04:24:00.805371+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823373+0000","last_change":"2023-10-09T04:24:00.823054+0000","last_active":"2023-10-09T04:24:00.823373+0000","last_peered":"2023-10-09T04:24:00.823373+0000","last_clean":"2023-10-09T04:24:00.823373+0000","last_became_active":"2023-10-09T04:24:00.822733+0000","last_became_peered":"2023-10-09T04:24:00.822733+0000","last_unstale":"2023-10-09T04:24:00.823373+0000","last_undegraded":"2023-10-09T04:24:00.823373+0000","last_fullsized":"2023-10-09T04:24:00.823373+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:13.369631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":648,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799128+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:23:59.799128+0000","last_peered":"2023-10-09T04:23:59.799128+0000","last_clean":"2023-10-09T04:23:59.799128+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:23:59.799128+0000","last_undegraded":"2023-10-09T04:23:59.799128+0000","last_fullsized":"2023-10-09T04:23:59.799128+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799465+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:23:59.799465+0000","last_peered":"2023-10-09T04:23:59.799465+0000","last_clean":"2023-10-09T04:23:59.799465+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:23:59.799465+0000","last_undegraded":"2023-10-09T04:23:59.799465+0000","last_fullsized":"2023-10-09T04:23:59.799465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.819863+0000","last_change":"2023-10-09T04:24:00.819863+0000","last_active":"2023-10-09T04:24:00.819863+0000","last_peered":"2023-10-09T04:24:00.819863+0000","last_clean":"2023-10-09T04:24:00.819863+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:24:00.819513+0000","last_unstale":"2023-10-09T04:24:00.819863+0000","last_undegraded":"2023-10-09T04:24:00.819863+0000","last_fullsized":"2023-10-09T04:24:00.819863+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:12.187156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798983+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:23:59.798983+0000","last_peered":"2023-10-09T04:23:59.798983+0000","last_clean":"2023-10-09T04:23:59.798983+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:23:59.798983+0000","last_undegraded":"2023-10-09T04:23:59.798983+0000","last_fullsized":"2023-10-09T04:23:59.798983+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":587,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792910+0000","last_change":"2023-10-09T04:23:18.767411+0000","last_active":"2023-10-09T04:23:59.792910+0000","last_peered":"2023-10-09T04:23:59.792910+0000","last_clean":"2023-10-09T04:23:59.792910+0000","last_became_active":"2023-10-09T04:23:18.766728+0000","last_became_peered":"2023-10-09T04:23:18.766728+0000","last_unstale":"2023-10-09T04:23:59.792910+0000","last_undegraded":"2023-10-09T04:23:59.792910+0000","last_fullsized":"2023-10-09T04:23:59.792910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":770,"num_read_kb":1698,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.803901+0000","last_change":"2023-10-09T04:22:50.696439+0000","last_active":"2023-10-09T04:24:00.803901+0000","last_peered":"2023-10-09T04:24:00.803901+0000","last_clean":"2023-10-09T04:24:00.803901+0000","last_became_active":"2023-10-09T04:22:50.696084+0000","last_became_peered":"2023-10-09T04:22:50.696084+0000","last_unstale":"2023-10-09T04:24:00.803901+0000","last_undegraded":"2023-10-09T04:24:00.803901+0000","last_fullsized":"2023-10-09T04:24:00.803901+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.931059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.807702+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:24:00.807702+0000","last_peered":"2023-10-09T04:24:00.807702+0000","last_clean":"2023-10-09T04:24:00.807702+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:24:00.807702+0000","last_undegraded":"2023-10-09T04:24:00.807702+0000","last_fullsized":"2023-10-09T04:24:00.807702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1209,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813355+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:24:00.813355+0000","last_peered":"2023-10-09T04:24:00.813355+0000","last_clean":"2023-10-09T04:24:00.813355+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:24:00.813355+0000","last_undegraded":"2023-10-09T04:24:00.813355+0000","last_fullsized":"2023-10-09T04:24:00.813355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814783+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:24:00.814783+0000","last_peered":"2023-10-09T04:24:00.814783+0000","last_clean":"2023-10-09T04:24:00.814783+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:24:00.814783+0000","last_undegraded":"2023-10-09T04:24:00.814783+0000","last_fullsized":"2023-10-09T04:24:00.814783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.801936+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:23:59.801936+0000","last_peered":"2023-10-09T04:23:59.801936+0000","last_clean":"2023-10-09T04:23:59.801936+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:23:59.801936+0000","last_undegraded":"2023-10-09T04:23:59.801936+0000","last_fullsized":"2023-10-09T04:23:59.801936+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":610,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792549+0000","last_change":"2023-10-09T04:22:50.705195+0000","last_active":"2023-10-09T04:23:59.792549+0000","last_peered":"2023-10-09T04:23:59.792549+0000","last_clean":"2023-10-09T04:23:59.792549+0000","last_became_active":"2023-10-09T04:22:50.704574+0000","last_became_peered":"2023-10-09T04:22:50.704574+0000","last_unstale":"2023-10-09T04:23:59.792549+0000","last_undegraded":"2023-10-09T04:23:59.792549+0000","last_fullsized":"2023-10-09T04:23:59.792549+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805058+0000","last_change":"2023-10-09T04:22:50.703397+0000","last_active":"2023-10-09T04:24:00.805058+0000","last_peered":"2023-10-09T04:24:00.805058+0000","last_clean":"2023-10-09T04:24:00.805058+0000","last_became_active":"2023-10-09T04:22:50.700260+0000","last_became_peered":"2023-10-09T04:22:50.700260+0000","last_unstale":"2023-10-09T04:24:00.805058+0000","last_undegraded":"2023-10-09T04:24:00.805058+0000","last_fullsized":"2023-10-09T04:24:00.805058+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:09.013765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813604+0000","last_change":"2023-10-09T04:24:00.813604+0000","last_active":"2023-10-09T04:24:00.813604+0000","last_peered":"2023-10-09T04:24:00.813604+0000","last_clean":"2023-10-09T04:24:00.813604+0000","last_became_active":"2023-10-09T04:24:00.813231+0000","last_became_peered":"2023-10-09T04:24:00.813231+0000","last_unstale":"2023-10-09T04:24:00.813604+0000","last_undegraded":"2023-10-09T04:24:00.813604+0000","last_fullsized":"2023-10-09T04:24:00.813604+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":473,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806172+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:24:00.806172+0000","last_peered":"2023-10-09T04:24:00.806172+0000","last_clean":"2023-10-09T04:24:00.806172+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:24:00.806172+0000","last_undegraded":"2023-10-09T04:24:00.806172+0000","last_fullsized":"2023-10-09T04:24:00.806172+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.825853+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:24:00.825853+0000","last_peered":"2023-10-09T04:24:00.825853+0000","last_clean":"2023-10-09T04:24:00.825853+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:24:00.825853+0000","last_undegraded":"2023-10-09T04:24:00.825853+0000","last_fullsized":"2023-10-09T04:24:00.825853+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822985+0000","last_change":"2023-10-09T04:24:00.822985+0000","last_active":"2023-10-09T04:24:00.822985+0000","last_peered":"2023-10-09T04:24:00.822985+0000","last_clean":"2023-10-09T04:24:00.822985+0000","last_became_active":"2023-10-09T04:24:00.822621+0000","last_became_peered":"2023-10-09T04:24:00.822621+0000","last_unstale":"2023-10-09T04:24:00.822985+0000","last_undegraded":"2023-10-09T04:24:00.822985+0000","last_fullsized":"2023-10-09T04:24:00.822985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:59.227822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802356+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:23:59.802356+0000","last_peered":"2023-10-09T04:23:59.802356+0000","last_clean":"2023-10-09T04:23:59.802356+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:23:59.802356+0000","last_undegraded":"2023-10-09T04:23:59.802356+0000","last_fullsized":"2023-10-09T04:23:59.802356+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802945+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:23:59.802945+0000","last_peered":"2023-10-09T04:23:59.802945+0000","last_clean":"2023-10-09T04:23:59.802945+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:23:59.802945+0000","last_undegraded":"2023-10-09T04:23:59.802945+0000","last_fullsized":"2023-10-09T04:23:59.802945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802914+0000","last_change":"2023-10-09T04:23:18.773293+0000","last_active":"2023-10-09T04:23:59.802914+0000","last_peered":"2023-10-09T04:23:59.802914+0000","last_clean":"2023-10-09T04:23:59.802914+0000","last_became_active":"2023-10-09T04:23:18.772947+0000","last_became_peered":"2023-10-09T04:23:18.772947+0000","last_unstale":"2023-10-09T04:23:59.802914+0000","last_undegraded":"2023-10-09T04:23:59.802914+0000","last_fullsized":"2023-10-09T04:23:59.802914+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799178+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:23:59.799178+0000","last_peered":"2023-10-09T04:23:59.799178+0000","last_clean":"2023-10-09T04:23:59.799178+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:23:59.799178+0000","last_undegraded":"2023-10-09T04:23:59.799178+0000","last_fullsized":"2023-10-09T04:23:59.799178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827521+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:24:00.827521+0000","last_peered":"2023-10-09T04:24:00.827521+0000","last_clean":"2023-10-09T04:24:00.827521+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:24:00.827521+0000","last_undegraded":"2023-10-09T04:24:00.827521+0000","last_fullsized":"2023-10-09T04:24:00.827521+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826876+0000","last_change":"2023-10-09T04:24:00.826876+0000","last_active":"2023-10-09T04:24:00.826876+0000","last_peered":"2023-10-09T04:24:00.826876+0000","last_clean":"2023-10-09T04:24:00.826876+0000","last_became_active":"2023-10-09T04:24:00.826469+0000","last_became_peered":"2023-10-09T04:24:00.826469+0000","last_unstale":"2023-10-09T04:24:00.826876+0000","last_undegraded":"2023-10-09T04:24:00.826876+0000","last_fullsized":"2023-10-09T04:24:00.826876+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:08.758269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804987+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:24:00.804987+0000","last_peered":"2023-10-09T04:24:00.804987+0000","last_clean":"2023-10-09T04:24:00.804987+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:24:00.804987+0000","last_undegraded":"2023-10-09T04:24:00.804987+0000","last_fullsized":"2023-10-09T04:24:00.804987+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822324+0000","last_change":"2023-10-09T04:24:00.822324+0000","last_active":"2023-10-09T04:24:00.822324+0000","last_peered":"2023-10-09T04:24:00.822324+0000","last_clean":"2023-10-09T04:24:00.822324+0000","last_became_active":"2023-10-09T04:24:00.821777+0000","last_became_peered":"2023-10-09T04:24:00.821777+0000","last_unstale":"2023-10-09T04:24:00.822324+0000","last_undegraded":"2023-10-09T04:24:00.822324+0000","last_fullsized":"2023-10-09T04:24:00.822324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798937+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:23:59.798937+0000","last_peered":"2023-10-09T04:23:59.798937+0000","last_clean":"2023-10-09T04:23:59.798937+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:23:59.798937+0000","last_undegraded":"2023-10-09T04:23:59.798937+0000","last_fullsized":"2023-10-09T04:23:59.798937+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797398+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:23:59.797398+0000","last_peered":"2023-10-09T04:23:59.797398+0000","last_clean":"2023-10-09T04:23:59.797398+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:23:59.797398+0000","last_undegraded":"2023-10-09T04:23:59.797398+0000","last_fullsized":"2023-10-09T04:23:59.797398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827553+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:24:00.827553+0000","last_peered":"2023-10-09T04:24:00.827553+0000","last_clean":"2023-10-09T04:24:00.827553+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:24:00.827553+0000","last_undegraded":"2023-10-09T04:24:00.827553+0000","last_fullsized":"2023-10-09T04:24:00.827553+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797418+0000","last_change":"2023-10-09T04:22:50.692658+0000","last_active":"2023-10-09T04:23:59.797418+0000","last_peered":"2023-10-09T04:23:59.797418+0000","last_clean":"2023-10-09T04:23:59.797418+0000","last_became_active":"2023-10-09T04:22:50.692027+0000","last_became_peered":"2023-10-09T04:22:50.692027+0000","last_unstale":"2023-10-09T04:23:59.797418+0000","last_undegraded":"2023-10-09T04:23:59.797418+0000","last_fullsized":"2023-10-09T04:23:59.797418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792610+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:23:59.792610+0000","last_peered":"2023-10-09T04:23:59.792610+0000","last_clean":"2023-10-09T04:23:59.792610+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:23:59.792610+0000","last_undegraded":"2023-10-09T04:23:59.792610+0000","last_fullsized":"2023-10-09T04:23:59.792610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797868+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:23:59.797868+0000","last_peered":"2023-10-09T04:23:59.797868+0000","last_clean":"2023-10-09T04:23:59.797868+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:23:59.797868+0000","last_undegraded":"2023-10-09T04:23:59.797868+0000","last_fullsized":"2023-10-09T04:23:59.797868+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822512+0000","last_change":"2023-10-09T04:24:00.822512+0000","last_active":"2023-10-09T04:24:00.822512+0000","last_peered":"2023-10-09T04:24:00.822512+0000","last_clean":"2023-10-09T04:24:00.822512+0000","last_became_active":"2023-10-09T04:24:00.822183+0000","last_became_peered":"2023-10-09T04:24:00.822183+0000","last_unstale":"2023-10-09T04:24:00.822512+0000","last_undegraded":"2023-10-09T04:24:00.822512+0000","last_fullsized":"2023-10-09T04:24:00.822512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.029015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799467+0000","last_change":"2023-10-09T04:22:23.612613+0000","last_active":"2023-10-09T04:23:59.799467+0000","last_peered":"2023-10-09T04:23:59.799467+0000","last_clean":"2023-10-09T04:23:59.799467+0000","last_became_active":"2023-10-09T04:22:23.611940+0000","last_became_peered":"2023-10-09T04:22:23.611940+0000","last_unstale":"2023-10-09T04:23:59.799467+0000","last_undegraded":"2023-10-09T04:23:59.799467+0000","last_fullsized":"2023-10-09T04:23:59.799467+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822959+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:24:00.822959+0000","last_peered":"2023-10-09T04:24:00.822959+0000","last_clean":"2023-10-09T04:24:00.822959+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:24:00.822959+0000","last_undegraded":"2023-10-09T04:24:00.822959+0000","last_fullsized":"2023-10-09T04:24:00.822959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.796146+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:23:59.796146+0000","last_peered":"2023-10-09T04:23:59.796146+0000","last_clean":"2023-10-09T04:23:59.796146+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:23:59.796146+0000","last_undegraded":"2023-10-09T04:23:59.796146+0000","last_fullsized":"2023-10-09T04:23:59.796146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822638+0000","last_change":"2023-10-09T04:24:00.822638+0000","last_active":"2023-10-09T04:24:00.822638+0000","last_peered":"2023-10-09T04:24:00.822638+0000","last_clean":"2023-10-09T04:24:00.822638+0000","last_became_active":"2023-10-09T04:24:00.822103+0000","last_became_peered":"2023-10-09T04:24:00.822103+0000","last_unstale":"2023-10-09T04:24:00.822638+0000","last_undegraded":"2023-10-09T04:24:00.822638+0000","last_fullsized":"2023-10-09T04:24:00.822638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826616+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:24:00.826616+0000","last_peered":"2023-10-09T04:24:00.826616+0000","last_clean":"2023-10-09T04:24:00.826616+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:24:00.826616+0000","last_undegraded":"2023-10-09T04:24:00.826616+0000","last_fullsized":"2023-10-09T04:24:00.826616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.821914+0000","last_change":"2023-10-09T04:24:00.821914+0000","last_active":"2023-10-09T04:24:00.821914+0000","last_peered":"2023-10-09T04:24:00.821914+0000","last_clean":"2023-10-09T04:24:00.821914+0000","last_became_active":"2023-10-09T04:24:00.821588+0000","last_became_peered":"2023-10-09T04:24:00.821588+0000","last_unstale":"2023-10-09T04:24:00.821914+0000","last_undegraded":"2023-10-09T04:24:00.821914+0000","last_fullsized":"2023-10-09T04:24:00.821914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:52.558769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797383+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:23:59.797383+0000","last_peered":"2023-10-09T04:23:59.797383+0000","last_clean":"2023-10-09T04:23:59.797383+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:23:59.797383+0000","last_undegraded":"2023-10-09T04:23:59.797383+0000","last_fullsized":"2023-10-09T04:23:59.797383+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802062+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:23:59.802062+0000","last_peered":"2023-10-09T04:23:59.802062+0000","last_clean":"2023-10-09T04:23:59.802062+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:23:59.802062+0000","last_undegraded":"2023-10-09T04:23:59.802062+0000","last_fullsized":"2023-10-09T04:23:59.802062+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802932+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:23:59.802932+0000","last_peered":"2023-10-09T04:23:59.802932+0000","last_clean":"2023-10-09T04:23:59.802932+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:23:59.802932+0000","last_undegraded":"2023-10-09T04:23:59.802932+0000","last_fullsized":"2023-10-09T04:23:59.802932+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805158+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:24:00.805158+0000","last_peered":"2023-10-09T04:24:00.805158+0000","last_clean":"2023-10-09T04:24:00.805158+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:24:00.805158+0000","last_undegraded":"2023-10-09T04:24:00.805158+0000","last_fullsized":"2023-10-09T04:24:00.805158+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824188+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:24:00.824188+0000","last_peered":"2023-10-09T04:24:00.824188+0000","last_clean":"2023-10-09T04:24:00.824188+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:24:00.824188+0000","last_undegraded":"2023-10-09T04:24:00.824188+0000","last_fullsized":"2023-10-09T04:24:00.824188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822436+0000","last_change":"2023-10-09T04:24:00.822436+0000","last_active":"2023-10-09T04:24:00.822436+0000","last_peered":"2023-10-09T04:24:00.822436+0000","last_clean":"2023-10-09T04:24:00.822436+0000","last_became_active":"2023-10-09T04:24:00.822108+0000","last_became_peered":"2023-10-09T04:24:00.822108+0000","last_unstale":"2023-10-09T04:24:00.822436+0000","last_undegraded":"2023-10-09T04:24:00.822436+0000","last_fullsized":"2023-10-09T04:24:00.822436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:24.737042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.793754+0000","last_change":"2023-10-09T04:22:50.698679+0000","last_active":"2023-10-09T04:23:59.793754+0000","last_peered":"2023-10-09T04:23:59.793754+0000","last_clean":"2023-10-09T04:23:59.793754+0000","last_became_active":"2023-10-09T04:22:50.698360+0000","last_became_peered":"2023-10-09T04:22:50.698360+0000","last_unstale":"2023-10-09T04:23:59.793754+0000","last_undegraded":"2023-10-09T04:23:59.793754+0000","last_fullsized":"2023-10-09T04:23:59.793754+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820037+0000","last_change":"2023-10-09T04:24:00.820037+0000","last_active":"2023-10-09T04:24:00.820037+0000","last_peered":"2023-10-09T04:24:00.820037+0000","last_clean":"2023-10-09T04:24:00.820037+0000","last_became_active":"2023-10-09T04:24:00.819650+0000","last_became_peered":"2023-10-09T04:24:00.819650+0000","last_unstale":"2023-10-09T04:24:00.820037+0000","last_undegraded":"2023-10-09T04:24:00.820037+0000","last_fullsized":"2023-10-09T04:24:00.820037+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:32.924825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.816756+0000","last_change":"2023-10-09T04:24:00.816214+0000","last_active":"2023-10-09T04:24:00.816756+0000","last_peered":"2023-10-09T04:24:00.816756+0000","last_clean":"2023-10-09T04:24:00.816756+0000","last_became_active":"2023-10-09T04:24:00.815867+0000","last_became_peered":"2023-10-09T04:24:00.815867+0000","last_unstale":"2023-10-09T04:24:00.816756+0000","last_undegraded":"2023-10-09T04:24:00.816756+0000","last_fullsized":"2023-10-09T04:24:00.816756+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:24.219649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799344+0000","last_change":"2023-10-09T04:22:50.702787+0000","last_active":"2023-10-09T04:23:59.799344+0000","last_peered":"2023-10-09T04:23:59.799344+0000","last_clean":"2023-10-09T04:23:59.799344+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:22:50.702392+0000","last_unstale":"2023-10-09T04:23:59.799344+0000","last_undegraded":"2023-10-09T04:23:59.799344+0000","last_fullsized":"2023-10-09T04:23:59.799344+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814141+0000","last_change":"2023-10-09T04:24:00.814141+0000","last_active":"2023-10-09T04:24:00.814141+0000","last_peered":"2023-10-09T04:24:00.814141+0000","last_clean":"2023-10-09T04:24:00.814141+0000","last_became_active":"2023-10-09T04:24:00.813799+0000","last_became_peered":"2023-10-09T04:24:00.813799+0000","last_unstale":"2023-10-09T04:24:00.814141+0000","last_undegraded":"2023-10-09T04:24:00.814141+0000","last_fullsized":"2023-10-09T04:24:00.814141+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:37.871798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822980+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:24:00.822980+0000","last_peered":"2023-10-09T04:24:00.822980+0000","last_clean":"2023-10-09T04:24:00.822980+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:24:00.822980+0000","last_undegraded":"2023-10-09T04:24:00.822980+0000","last_fullsized":"2023-10-09T04:24:00.822980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826025+0000","last_change":"2023-10-09T04:24:00.825647+0000","last_active":"2023-10-09T04:24:00.826025+0000","last_peered":"2023-10-09T04:24:00.826025+0000","last_clean":"2023-10-09T04:24:00.826025+0000","last_became_active":"2023-10-09T04:24:00.825310+0000","last_became_peered":"2023-10-09T04:24:00.825310+0000","last_unstale":"2023-10-09T04:24:00.826025+0000","last_undegraded":"2023-10-09T04:24:00.826025+0000","last_fullsized":"2023-10-09T04:24:00.826025+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:07.180568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826023+0000","last_change":"2023-10-09T04:24:00.826023+0000","last_active":"2023-10-09T04:24:00.826023+0000","last_peered":"2023-10-09T04:24:00.826023+0000","last_clean":"2023-10-09T04:24:00.826023+0000","last_became_active":"2023-10-09T04:24:00.825466+0000","last_became_peered":"2023-10-09T04:24:00.825466+0000","last_unstale":"2023-10-09T04:24:00.826023+0000","last_undegraded":"2023-10-09T04:24:00.826023+0000","last_fullsized":"2023-10-09T04:24:00.826023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:39.932991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.800944+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:23:59.800944+0000","last_peered":"2023-10-09T04:23:59.800944+0000","last_clean":"2023-10-09T04:23:59.800944+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:23:59.800944+0000","last_undegraded":"2023-10-09T04:23:59.800944+0000","last_fullsized":"2023-10-09T04:23:59.800944+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430419,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.56499999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92400000000000004,"15min":0.91600000000000004},"min":{"1min":0.624,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.212,"5min":1.375,"15min":1.375},"last":0.81999999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.59099999999999997,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.1719999999999999,"5min":2.097,"15min":2.097},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.94499999999999995,"15min":0.93899999999999995},"min":{"1min":0.625,"5min":0.503,"15min":0.503},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.089}]},{"osd":2,"last update":"Mon Oct 9 04:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.69499999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1040000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.98399999999999999,"15min":0.98899999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.242,"15min":1.242},"last":1.077}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.749,"5min":0.749,"15min":0.749},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.98399999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91000000000000003,"15min":0.91300000000000003},"min":{"1min":0.67100000000000004,"5min":0.55000000000000004,"15min":0.503},"max":{"1min":1.1399999999999999,"5min":1.4550000000000001,"15min":2.0419999999999998},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.70199999999999996,"5min":0.55200000000000005,"15min":0.495},"max":{"1min":1.2190000000000001,"5min":1.4399999999999999,"15min":2.1139999999999999},"last":0.72499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.97099999999999997,"15min":0.98299999999999998},"min":{"1min":0.74199999999999999,"5min":0.51500000000000001,"15min":0.497},"max":{"1min":1.329,"5min":1.393,"15min":1.9510000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0329999999999999,"15min":1.024},"min":{"1min":0.83099999999999996,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.3160000000000001,"5min":1.363,"15min":2.2349999999999999},"last":1.21}]},{"osd":6,"last update":"Mon Oct 9 04:25:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.046,"15min":1.0589999999999999},"min":{"1min":0.95399999999999996,"5min":0.51600000000000001,"15min":0.46100000000000002},"max":{"1min":1.5,"5min":1.5,"15min":2.3100000000000001},"last":1.163},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.044,"15min":1.044},"min":{"1min":0.77700000000000002,"5min":0.496,"15min":0.48199999999999998},"max":{"1min":1.248,"5min":1.4750000000000001,"15min":2.1909999999999998},"last":1.0289999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298728,"kb_used_data":1632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446424,"statfs":{"total":95995035648,"available":95689138176,"internally_reserved":0,"allocated":1671168,"data_stored":1171554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.88700000000000001,"15min":0.89100000000000001},"min":{"1min":0.57299999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2270000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.78700000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.88600000000000001,"15min":0.89200000000000002},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1619999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91200000000000003,"15min":0.90200000000000002},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.1419999999999999,"5min":1.587,"15min":1.587},"last":0.88900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.95399999999999996,"15min":0.96499999999999997},"min":{"1min":0.64000000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.204,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.2250000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.82899999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.04,"15min":1.04},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.2230000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93700000000000006,"15min":0.91700000000000004},"min":{"1min":0.56799999999999995,"5min":0.51800000000000002,"15min":0.40899999999999997},"max":{"1min":1.147,"5min":1.367,"15min":1.663},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.99199999999999999,"15min":0.95799999999999996},"min":{"1min":0.65900000000000003,"5min":0.51200000000000001,"15min":0.443},"max":{"1min":1.319,"5min":1.405,"15min":1.548},"last":0.69899999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0149999999999999,"15min":1.0369999999999999},"min":{"1min":0.67700000000000005,"5min":0.55900000000000005,"15min":0.50700000000000001},"max":{"1min":1.3009999999999999,"5min":1.498,"15min":3.9180000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0289999999999999,"15min":1.0049999999999999},"min":{"1min":0.745,"5min":0.56100000000000005,"15min":0.505},"max":{"1min":1.278,"5min":1.3320000000000001,"15min":1.635},"last":0.95799999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:25:13 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0860000000000001,"15min":1.0720000000000001},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.626},"max":{"1min":1.2609999999999999,"5min":1.5569999999999999,"15min":1.9570000000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1679999999999999,"15min":1.159},"min":{"1min":0.85399999999999998,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3149999999999999,"5min":1.6279999999999999,"15min":1.9990000000000001},"last":1.135}]}]},{"osd":1,"up_from":93,"seq":399431958568,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":6,"kb_used_meta":31545,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":585588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":32302092},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.66600000000000004,"15min":0.67100000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.90500000000000003,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.72299999999999998,"15min":0.73899999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.93200000000000005,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.68500000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:25:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.66400000000000003,"15min":0.67300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.76700000000000002,"15min":0.77900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.234,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.86799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":1.038},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.123,"5min":1.123,"15min":1.123},"last":0.60699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.91200000000000003,"15min":0.91800000000000004},"min":{"1min":0.61199999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1759999999999999,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.024,"15min":1.0389999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.333,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.093}]},{"osd":5,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.99199999999999999,"15min":1.01},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.4550000000000001,"5min":1.819,"15min":1.819},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.97899999999999998,"15min":0.98499999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3009999999999999,"5min":1.595,"15min":1.595},"last":0.92300000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.016,"15min":1.026},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.5449999999999999,"5min":1.877,"15min":1.877},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.0009999999999999,"15min":1.0169999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2769999999999999,"5min":1.724,"15min":1.724},"last":1.0149999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0029999999999999,"15min":1.004},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.349,"5min":1.653,"15min":1.653},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0920000000000001,"15min":1.1120000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.4019999999999999,"5min":1.786,"15min":1.786},"last":0.97299999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122094,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33304,"kb_used_data":1680,"kb_used_omap":9,"kb_used_meta":31606,"kb_avail":93711848,"statfs":{"total":95995035648,"available":95960932352,"internally_reserved":0,"allocated":1720320,"data_stored":1178304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":32365041},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.63300000000000001,"15min":0.64100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.879},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.64800000000000002,"15min":0.66400000000000003},"min":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80700000000000005,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.70699999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.75700000000000001,"15min":0.77500000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":1.069}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":1.131},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.93300000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0249999999999999,"15min":1.0720000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1180000000000001,"5min":2.036,"15min":2.036},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.0209999999999999,"15min":1.0349999999999999},"min":{"1min":0.69499999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.25,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.244}]},{"osd":5,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.302,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":1.2929999999999999},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.99299999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1379999999999999,"5min":1.613,"15min":1.613},"last":1.046}]},{"osd":6,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1,"15min":1.028},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2769999999999999,"5min":1.712,"15min":1.712},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.012,"15min":1.012},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2869999999999999,"5min":1.641,"15min":1.641},"last":1.1850000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.093,"15min":1.123},"min":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2649999999999999,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":1.157},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.101,"15min":1.137},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1799999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":1.1000000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795043,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36728,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31540,"kb_avail":93708424,"statfs":{"total":95995035648,"available":95957426176,"internally_reserved":0,"allocated":5292032,"data_stored":4777245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":32297002},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.66700000000000004,"15min":0.69099999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.91000000000000003,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.79700000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.72399999999999998,"15min":0.73599999999999999},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.70999999999999996,"15min":0.71199999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.91700000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:25:01 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":2.2759999999999998,"5min":2.2759999999999998,"15min":2.2759999999999998},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.1899999999999999,"5min":2.1899999999999999,"15min":2.1899999999999999},"last":0.97799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94299999999999995,"15min":0.94599999999999995},"min":{"1min":0.66100000000000003,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.083,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.1339999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.94399999999999995,"15min":0.95799999999999996},"min":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1579999999999999,"5min":1.429,"15min":1.429},"last":1.083}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.024},"min":{"1min":0.70099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2989999999999999,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.262,"5min":1.718,"15min":1.718},"last":1.1579999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.006,"15min":1.02},"min":{"1min":0.60399999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3440000000000001,"5min":1.855,"15min":1.855},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.05,"15min":1.0649999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2769999999999999,"5min":1.603,"15min":1.603},"last":1.0169999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.115,"15min":1.139},"min":{"1min":0.63900000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3480000000000001,"5min":1.825,"15min":1.825},"last":1.111},{"interface":"front","average":{"1min":0.995,"5min":1.077,"15min":1.091},"min":{"1min":0.67900000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2949999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":1.0609999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631514,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32552,"kb_used_data":1056,"kb_used_omap":5,"kb_used_meta":31482,"kb_avail":93712600,"statfs":{"total":95995035648,"available":95961702400,"internally_reserved":0,"allocated":1081344,"data_stored":580896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":32237742},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":1.2709999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.70899999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.873}]},{"osd":4,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":2.0470000000000002,"5min":2.0470000000000002,"15min":2.0470000000000002},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.83599999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.224},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.1970000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"min":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.133},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.0149999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.145,"15min":1.145},"min":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.095},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"min":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"max":{"1min":2.3980000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":1.165}]}]},{"osd":4,"up_from":30,"seq":128849019116,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84699999999999998,"15min":0.85699999999999998},"min":{"1min":0.54000000000000004,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.165,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0629999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.99099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.88700000000000001,"15min":0.90400000000000003},"min":{"1min":0.54900000000000004,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1439999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.748},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0269999999999999,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.86799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.629,"5min":0.502,"15min":0.502},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96099999999999997,"15min":0.95999999999999996},"min":{"1min":0.64100000000000001,"5min":0.623,"15min":0.623},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.96599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.085},{"interface":"front","average":{"1min":1.087,"5min":1.087,"15min":1.087},"min":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.02}]},{"osd":5,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.95499999999999996,"15min":0.93999999999999995},"min":{"1min":0.59999999999999998,"5min":0.56399999999999995,"15min":0.433},"max":{"1min":1.266,"5min":1.5269999999999999,"15min":1.5509999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.93700000000000006,"15min":0.90500000000000003},"min":{"1min":0.69999999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.1970000000000001,"5min":1.877,"15min":1.877},"last":0.94299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.99099999999999999,"15min":0.96299999999999997},"min":{"1min":0.755,"5min":0.56100000000000005,"15min":0.47099999999999997},"max":{"1min":1.389,"5min":1.389,"15min":1.7290000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.02,"5min":1.0549999999999999,"15min":1.026},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.46200000000000002},"max":{"1min":1.3740000000000001,"5min":1.583,"15min":1.796},"last":0.89700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.157,"15min":1.125},"min":{"1min":0.82999999999999996,"5min":0.82599999999999996,"15min":0.65900000000000003},"max":{"1min":1.5409999999999999,"5min":1.653,"15min":1.8600000000000001},"last":1.169},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1319999999999999,"15min":1.1080000000000001},"min":{"1min":0.88800000000000001,"5min":0.76500000000000001,"15min":0.57699999999999996},"max":{"1min":1.3819999999999999,"5min":1.512,"15min":1.923},"last":1.2150000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822883,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302232,"kb_used_data":5136,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442920,"statfs":{"total":95995035648,"available":95685550080,"internally_reserved":0,"allocated":5259264,"data_stored":4770995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.87,"15min":0.872},"min":{"1min":0.63600000000000001,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.1399999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.74399999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.0920000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:25:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.97399999999999998,"15min":0.995},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2549999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":1.012},{"interface":"front","average":{"1min":1.022,"5min":0.96399999999999997,"15min":0.95399999999999996},"min":{"1min":0.71099999999999997,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.4019999999999999,"5min":1.645,"15min":1.645},"last":1.141}]},{"osd":2,"last update":"Mon Oct 9 04:25:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93100000000000005,"15min":0.93600000000000005},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0189999999999999,"15min":1.0209999999999999},"min":{"1min":0.79700000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.266,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.87}]},{"osd":3,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.069},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.05},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.028}]},{"osd":4,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.93600000000000005,"15min":0.94699999999999995},"min":{"1min":0.70999999999999996,"5min":0.51600000000000001,"15min":0.47099999999999997},"max":{"1min":1.383,"5min":1.383,"15min":2.165},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.075,"5min":1.0429999999999999,"15min":0.997},"min":{"1min":0.72599999999999998,"5min":0.54600000000000004,"15min":0.47599999999999998},"max":{"1min":1.4319999999999999,"5min":3.7989999999999999,"15min":3.7989999999999999},"last":0.96699999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0229999999999999,"15min":1.0069999999999999},"min":{"1min":0.77000000000000002,"5min":0.50700000000000001,"15min":0.40899999999999997},"max":{"1min":1.5700000000000001,"5min":2.2999999999999998,"15min":2.2999999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.083,"15min":1.0720000000000001},"min":{"1min":0.68500000000000005,"5min":0.58799999999999997,"15min":0.44800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":2.5840000000000001},"last":0.93300000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.1100000000000001,"15min":1.1040000000000001},"min":{"1min":0.75,"5min":0.68899999999999995,"15min":0.42599999999999999},"max":{"1min":1.3540000000000001,"5min":1.6160000000000001,"15min":2.7050000000000001},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0820000000000001,"15min":1.0980000000000001},"min":{"1min":0.753,"5min":0.73999999999999999,"15min":0.47799999999999998},"max":{"1min":1.456,"5min":1.456,"15min":2.2829999999999999},"last":0.98299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:26:05.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:26:06.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:06 smithi082 bash[79733]: audit 2023-10-09T04:26:05.089177+0000 mgr.y (mgr.34107) 348 : audit 0 from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:06 smithi082 bash[79733]: cluster 2023-10-09T04:26:05.325808+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:06.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:06 smithi082 bash[77063]: audit 2023-10-09T04:26:05.089177+0000 mgr.y (mgr.34107) 348 : audit 0 from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:06.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:06 smithi082 bash[77063]: cluster 2023-10-09T04:26:05.325808+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:06.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:06 smithi153 bash[59565]: audit 2023-10-09T04:26:05.089177+0000 mgr.y (mgr.34107) 348 : audit 0 from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:06.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:06 smithi153 bash[59565]: cluster 2023-10-09T04:26:05.325808+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:08.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:08 smithi082 bash[77063]: cluster 2023-10-09T04:26:07.326520+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:08.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:08 smithi082 bash[79733]: cluster 2023-10-09T04:26:07.326520+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:08.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:08 smithi153 bash[59565]: cluster 2023-10-09T04:26:07.326520+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:09] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:26:10.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:10 smithi153 bash[59565]: cluster 2023-10-09T04:26:09.327315+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:10 smithi082 bash[77063]: cluster 2023-10-09T04:26:09.327315+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:10 smithi082 bash[79733]: cluster 2023-10-09T04:26:09.327315+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:10.751 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:12.641 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:12 smithi082 bash[77063]: cluster 2023-10-09T04:26:11.328105+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:12.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:12 smithi082 bash[79733]: cluster 2023-10-09T04:26:11.328105+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:12 smithi153 bash[59565]: cluster 2023-10-09T04:26:11.328105+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:13.963 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:26:13.966 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:26:14.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:14 smithi082 bash[79733]: cluster 2023-10-09T04:26:13.328769+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:14.499 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:14 smithi082 bash[79733]: audit 2023-10-09T04:26:14.254952+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:14.499 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:14 smithi082 bash[77063]: cluster 2023-10-09T04:26:13.328769+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:14.500 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:14 smithi082 bash[77063]: audit 2023-10-09T04:26:14.254952+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:14 smithi153 bash[59565]: cluster 2023-10-09T04:26:13.328769+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:14 smithi153 bash[59565]: audit 2023-10-09T04:26:14.254952+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:14.733 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":285,"stamp":"2023-10-09T04:26:13.328335+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341268,"kb_used_data":22548,"kb_used_omap":34,"kb_used_meta":1318621,"kb_avail":748619948,"statfs":{"total":767960285184,"available":766586826752,"internally_reserved":0,"allocated":23089152,"data_stored":19011810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1350268629},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004795"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802015+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:23:59.802015+0000","last_peered":"2023-10-09T04:23:59.802015+0000","last_clean":"2023-10-09T04:23:59.802015+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:23:59.802015+0000","last_undegraded":"2023-10-09T04:23:59.802015+0000","last_fullsized":"2023-10-09T04:23:59.802015+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804059+0000","last_change":"2023-10-09T04:23:18.765967+0000","last_active":"2023-10-09T04:24:00.804059+0000","last_peered":"2023-10-09T04:24:00.804059+0000","last_clean":"2023-10-09T04:24:00.804059+0000","last_became_active":"2023-10-09T04:23:18.765587+0000","last_became_peered":"2023-10-09T04:23:18.765587+0000","last_unstale":"2023-10-09T04:24:00.804059+0000","last_undegraded":"2023-10-09T04:24:00.804059+0000","last_fullsized":"2023-10-09T04:24:00.804059+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:10.976153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.815714+0000","last_change":"2023-10-09T04:24:00.815714+0000","last_active":"2023-10-09T04:24:00.815714+0000","last_peered":"2023-10-09T04:24:00.815714+0000","last_clean":"2023-10-09T04:24:00.815714+0000","last_became_active":"2023-10-09T04:24:00.815389+0000","last_became_peered":"2023-10-09T04:24:00.815389+0000","last_unstale":"2023-10-09T04:24:00.815714+0000","last_undegraded":"2023-10-09T04:24:00.815714+0000","last_fullsized":"2023-10-09T04:24:00.815714+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:39.857138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":80,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820442+0000","last_change":"2023-10-09T04:24:00.819974+0000","last_active":"2023-10-09T04:24:00.820442+0000","last_peered":"2023-10-09T04:24:00.820442+0000","last_clean":"2023-10-09T04:24:00.820442+0000","last_became_active":"2023-10-09T04:24:00.819558+0000","last_became_peered":"2023-10-09T04:24:00.819558+0000","last_unstale":"2023-10-09T04:24:00.820442+0000","last_undegraded":"2023-10-09T04:24:00.820442+0000","last_fullsized":"2023-10-09T04:24:00.820442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827513+0000","last_change":"2023-10-09T04:24:00.827513+0000","last_active":"2023-10-09T04:24:00.827513+0000","last_peered":"2023-10-09T04:24:00.827513+0000","last_clean":"2023-10-09T04:24:00.827513+0000","last_became_active":"2023-10-09T04:24:00.827037+0000","last_became_peered":"2023-10-09T04:24:00.827037+0000","last_unstale":"2023-10-09T04:24:00.827513+0000","last_undegraded":"2023-10-09T04:24:00.827513+0000","last_fullsized":"2023-10-09T04:24:00.827513+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794630+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:23:59.794630+0000","last_peered":"2023-10-09T04:23:59.794630+0000","last_clean":"2023-10-09T04:23:59.794630+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:23:59.794630+0000","last_undegraded":"2023-10-09T04:23:59.794630+0000","last_fullsized":"2023-10-09T04:23:59.794630+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827442+0000","last_change":"2023-10-09T04:24:00.827442+0000","last_active":"2023-10-09T04:24:00.827442+0000","last_peered":"2023-10-09T04:24:00.827442+0000","last_clean":"2023-10-09T04:24:00.827442+0000","last_became_active":"2023-10-09T04:24:00.827036+0000","last_became_peered":"2023-10-09T04:24:00.827036+0000","last_unstale":"2023-10-09T04:24:00.827442+0000","last_undegraded":"2023-10-09T04:24:00.827442+0000","last_fullsized":"2023-10-09T04:24:00.827442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.794088+0000","last_change":"2023-10-09T04:22:50.701405+0000","last_active":"2023-10-09T04:23:59.794088+0000","last_peered":"2023-10-09T04:23:59.794088+0000","last_clean":"2023-10-09T04:23:59.794088+0000","last_became_active":"2023-10-09T04:22:50.700948+0000","last_became_peered":"2023-10-09T04:22:50.700948+0000","last_unstale":"2023-10-09T04:23:59.794088+0000","last_undegraded":"2023-10-09T04:23:59.794088+0000","last_fullsized":"2023-10-09T04:23:59.794088+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813394+0000","last_change":"2023-10-09T04:24:00.811849+0000","last_active":"2023-10-09T04:24:00.813394+0000","last_peered":"2023-10-09T04:24:00.813394+0000","last_clean":"2023-10-09T04:24:00.813394+0000","last_became_active":"2023-10-09T04:24:00.811528+0000","last_became_peered":"2023-10-09T04:24:00.811528+0000","last_unstale":"2023-10-09T04:24:00.813394+0000","last_undegraded":"2023-10-09T04:24:00.813394+0000","last_fullsized":"2023-10-09T04:24:00.813394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:25.981562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":42,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805109+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:24:00.805109+0000","last_peered":"2023-10-09T04:24:00.805109+0000","last_clean":"2023-10-09T04:24:00.805109+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:24:00.805109+0000","last_undegraded":"2023-10-09T04:24:00.805109+0000","last_fullsized":"2023-10-09T04:24:00.805109+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802722+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:23:59.802722+0000","last_peered":"2023-10-09T04:23:59.802722+0000","last_clean":"2023-10-09T04:23:59.802722+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:23:59.802722+0000","last_undegraded":"2023-10-09T04:23:59.802722+0000","last_fullsized":"2023-10-09T04:23:59.802722+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805271+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:24:00.805271+0000","last_peered":"2023-10-09T04:24:00.805271+0000","last_clean":"2023-10-09T04:24:00.805271+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:24:00.805271+0000","last_undegraded":"2023-10-09T04:24:00.805271+0000","last_fullsized":"2023-10-09T04:24:00.805271+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":151,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799287+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:23:59.799287+0000","last_peered":"2023-10-09T04:23:59.799287+0000","last_clean":"2023-10-09T04:23:59.799287+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:23:59.799287+0000","last_undegraded":"2023-10-09T04:23:59.799287+0000","last_fullsized":"2023-10-09T04:23:59.799287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823561+0000","last_change":"2023-10-09T04:24:00.822119+0000","last_active":"2023-10-09T04:24:00.823561+0000","last_peered":"2023-10-09T04:24:00.823561+0000","last_clean":"2023-10-09T04:24:00.823561+0000","last_became_active":"2023-10-09T04:24:00.821789+0000","last_became_peered":"2023-10-09T04:24:00.821789+0000","last_unstale":"2023-10-09T04:24:00.823561+0000","last_undegraded":"2023-10-09T04:24:00.823561+0000","last_fullsized":"2023-10-09T04:24:00.823561+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:09.080969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":324,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804070+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:24:00.804070+0000","last_peered":"2023-10-09T04:24:00.804070+0000","last_clean":"2023-10-09T04:24:00.804070+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:24:00.804070+0000","last_undegraded":"2023-10-09T04:24:00.804070+0000","last_fullsized":"2023-10-09T04:24:00.804070+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827582+0000","last_change":"2023-10-09T04:24:00.824546+0000","last_active":"2023-10-09T04:24:00.827582+0000","last_peered":"2023-10-09T04:24:00.827582+0000","last_clean":"2023-10-09T04:24:00.827582+0000","last_became_active":"2023-10-09T04:24:00.824005+0000","last_became_peered":"2023-10-09T04:24:00.824005+0000","last_unstale":"2023-10-09T04:24:00.827582+0000","last_undegraded":"2023-10-09T04:24:00.827582+0000","last_fullsized":"2023-10-09T04:24:00.827582+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:46.407835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804761+0000","last_change":"2023-10-09T04:22:23.611501+0000","last_active":"2023-10-09T04:24:00.804761+0000","last_peered":"2023-10-09T04:24:00.804761+0000","last_clean":"2023-10-09T04:24:00.804761+0000","last_became_active":"2023-10-09T04:22:23.611116+0000","last_became_peered":"2023-10-09T04:22:23.611116+0000","last_unstale":"2023-10-09T04:24:00.804761+0000","last_undegraded":"2023-10-09T04:24:00.804761+0000","last_fullsized":"2023-10-09T04:24:00.804761+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:19.031531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806273+0000","last_change":"2023-10-09T04:23:18.773712+0000","last_active":"2023-10-09T04:24:00.806273+0000","last_peered":"2023-10-09T04:24:00.806273+0000","last_clean":"2023-10-09T04:24:00.806273+0000","last_became_active":"2023-10-09T04:23:18.773331+0000","last_became_peered":"2023-10-09T04:23:18.773331+0000","last_unstale":"2023-10-09T04:24:00.806273+0000","last_undegraded":"2023-10-09T04:24:00.806273+0000","last_fullsized":"2023-10-09T04:24:00.806273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:53.528302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":317,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797257+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:23:59.797257+0000","last_peered":"2023-10-09T04:23:59.797257+0000","last_clean":"2023-10-09T04:23:59.797257+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:23:59.797257+0000","last_undegraded":"2023-10-09T04:23:59.797257+0000","last_fullsized":"2023-10-09T04:23:59.797257+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805371+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:24:00.805371+0000","last_peered":"2023-10-09T04:24:00.805371+0000","last_clean":"2023-10-09T04:24:00.805371+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:24:00.805371+0000","last_undegraded":"2023-10-09T04:24:00.805371+0000","last_fullsized":"2023-10-09T04:24:00.805371+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823373+0000","last_change":"2023-10-09T04:24:00.823054+0000","last_active":"2023-10-09T04:24:00.823373+0000","last_peered":"2023-10-09T04:24:00.823373+0000","last_clean":"2023-10-09T04:24:00.823373+0000","last_became_active":"2023-10-09T04:24:00.822733+0000","last_became_peered":"2023-10-09T04:24:00.822733+0000","last_unstale":"2023-10-09T04:24:00.823373+0000","last_undegraded":"2023-10-09T04:24:00.823373+0000","last_fullsized":"2023-10-09T04:24:00.823373+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:13.369631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":648,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799128+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:23:59.799128+0000","last_peered":"2023-10-09T04:23:59.799128+0000","last_clean":"2023-10-09T04:23:59.799128+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:23:59.799128+0000","last_undegraded":"2023-10-09T04:23:59.799128+0000","last_fullsized":"2023-10-09T04:23:59.799128+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799465+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:23:59.799465+0000","last_peered":"2023-10-09T04:23:59.799465+0000","last_clean":"2023-10-09T04:23:59.799465+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:23:59.799465+0000","last_undegraded":"2023-10-09T04:23:59.799465+0000","last_fullsized":"2023-10-09T04:23:59.799465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820447+0000","last_change":"2023-10-09T04:24:00.819863+0000","last_active":"2023-10-09T04:24:00.820447+0000","last_peered":"2023-10-09T04:24:00.820447+0000","last_clean":"2023-10-09T04:24:00.820447+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:24:00.819513+0000","last_unstale":"2023-10-09T04:24:00.820447+0000","last_undegraded":"2023-10-09T04:24:00.820447+0000","last_fullsized":"2023-10-09T04:24:00.820447+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:12.187156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":306,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798983+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:23:59.798983+0000","last_peered":"2023-10-09T04:23:59.798983+0000","last_clean":"2023-10-09T04:23:59.798983+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:23:59.798983+0000","last_undegraded":"2023-10-09T04:23:59.798983+0000","last_fullsized":"2023-10-09T04:23:59.798983+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":587,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792910+0000","last_change":"2023-10-09T04:23:18.767411+0000","last_active":"2023-10-09T04:23:59.792910+0000","last_peered":"2023-10-09T04:23:59.792910+0000","last_clean":"2023-10-09T04:23:59.792910+0000","last_became_active":"2023-10-09T04:23:18.766728+0000","last_became_peered":"2023-10-09T04:23:18.766728+0000","last_unstale":"2023-10-09T04:23:59.792910+0000","last_undegraded":"2023-10-09T04:23:59.792910+0000","last_fullsized":"2023-10-09T04:23:59.792910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":770,"num_read_kb":1698,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.803901+0000","last_change":"2023-10-09T04:22:50.696439+0000","last_active":"2023-10-09T04:24:00.803901+0000","last_peered":"2023-10-09T04:24:00.803901+0000","last_clean":"2023-10-09T04:24:00.803901+0000","last_became_active":"2023-10-09T04:22:50.696084+0000","last_became_peered":"2023-10-09T04:22:50.696084+0000","last_unstale":"2023-10-09T04:24:00.803901+0000","last_undegraded":"2023-10-09T04:24:00.803901+0000","last_fullsized":"2023-10-09T04:24:00.803901+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.931059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.807702+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:24:00.807702+0000","last_peered":"2023-10-09T04:24:00.807702+0000","last_clean":"2023-10-09T04:24:00.807702+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:24:00.807702+0000","last_undegraded":"2023-10-09T04:24:00.807702+0000","last_fullsized":"2023-10-09T04:24:00.807702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1209,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813355+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:24:00.813355+0000","last_peered":"2023-10-09T04:24:00.813355+0000","last_clean":"2023-10-09T04:24:00.813355+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:24:00.813355+0000","last_undegraded":"2023-10-09T04:24:00.813355+0000","last_fullsized":"2023-10-09T04:24:00.813355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814783+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:24:00.814783+0000","last_peered":"2023-10-09T04:24:00.814783+0000","last_clean":"2023-10-09T04:24:00.814783+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:24:00.814783+0000","last_undegraded":"2023-10-09T04:24:00.814783+0000","last_fullsized":"2023-10-09T04:24:00.814783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.801936+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:23:59.801936+0000","last_peered":"2023-10-09T04:23:59.801936+0000","last_clean":"2023-10-09T04:23:59.801936+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:23:59.801936+0000","last_undegraded":"2023-10-09T04:23:59.801936+0000","last_fullsized":"2023-10-09T04:23:59.801936+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":610,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792549+0000","last_change":"2023-10-09T04:22:50.705195+0000","last_active":"2023-10-09T04:23:59.792549+0000","last_peered":"2023-10-09T04:23:59.792549+0000","last_clean":"2023-10-09T04:23:59.792549+0000","last_became_active":"2023-10-09T04:22:50.704574+0000","last_became_peered":"2023-10-09T04:22:50.704574+0000","last_unstale":"2023-10-09T04:23:59.792549+0000","last_undegraded":"2023-10-09T04:23:59.792549+0000","last_fullsized":"2023-10-09T04:23:59.792549+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805058+0000","last_change":"2023-10-09T04:22:50.703397+0000","last_active":"2023-10-09T04:24:00.805058+0000","last_peered":"2023-10-09T04:24:00.805058+0000","last_clean":"2023-10-09T04:24:00.805058+0000","last_became_active":"2023-10-09T04:22:50.700260+0000","last_became_peered":"2023-10-09T04:22:50.700260+0000","last_unstale":"2023-10-09T04:24:00.805058+0000","last_undegraded":"2023-10-09T04:24:00.805058+0000","last_fullsized":"2023-10-09T04:24:00.805058+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:09.013765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.813604+0000","last_change":"2023-10-09T04:24:00.813604+0000","last_active":"2023-10-09T04:24:00.813604+0000","last_peered":"2023-10-09T04:24:00.813604+0000","last_clean":"2023-10-09T04:24:00.813604+0000","last_became_active":"2023-10-09T04:24:00.813231+0000","last_became_peered":"2023-10-09T04:24:00.813231+0000","last_unstale":"2023-10-09T04:24:00.813604+0000","last_undegraded":"2023-10-09T04:24:00.813604+0000","last_fullsized":"2023-10-09T04:24:00.813604+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":473,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.806172+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:24:00.806172+0000","last_peered":"2023-10-09T04:24:00.806172+0000","last_clean":"2023-10-09T04:24:00.806172+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:24:00.806172+0000","last_undegraded":"2023-10-09T04:24:00.806172+0000","last_fullsized":"2023-10-09T04:24:00.806172+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.825853+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:24:00.825853+0000","last_peered":"2023-10-09T04:24:00.825853+0000","last_clean":"2023-10-09T04:24:00.825853+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:24:00.825853+0000","last_undegraded":"2023-10-09T04:24:00.825853+0000","last_fullsized":"2023-10-09T04:24:00.825853+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":75,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.823640+0000","last_change":"2023-10-09T04:24:00.822985+0000","last_active":"2023-10-09T04:24:00.823640+0000","last_peered":"2023-10-09T04:24:00.823640+0000","last_clean":"2023-10-09T04:24:00.823640+0000","last_became_active":"2023-10-09T04:24:00.822621+0000","last_became_peered":"2023-10-09T04:24:00.822621+0000","last_unstale":"2023-10-09T04:24:00.823640+0000","last_undegraded":"2023-10-09T04:24:00.823640+0000","last_fullsized":"2023-10-09T04:24:00.823640+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:59.227822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802356+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:23:59.802356+0000","last_peered":"2023-10-09T04:23:59.802356+0000","last_clean":"2023-10-09T04:23:59.802356+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:23:59.802356+0000","last_undegraded":"2023-10-09T04:23:59.802356+0000","last_fullsized":"2023-10-09T04:23:59.802356+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802945+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:23:59.802945+0000","last_peered":"2023-10-09T04:23:59.802945+0000","last_clean":"2023-10-09T04:23:59.802945+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:23:59.802945+0000","last_undegraded":"2023-10-09T04:23:59.802945+0000","last_fullsized":"2023-10-09T04:23:59.802945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802914+0000","last_change":"2023-10-09T04:23:18.773293+0000","last_active":"2023-10-09T04:23:59.802914+0000","last_peered":"2023-10-09T04:23:59.802914+0000","last_clean":"2023-10-09T04:23:59.802914+0000","last_became_active":"2023-10-09T04:23:18.772947+0000","last_became_peered":"2023-10-09T04:23:18.772947+0000","last_unstale":"2023-10-09T04:23:59.802914+0000","last_undegraded":"2023-10-09T04:23:59.802914+0000","last_fullsized":"2023-10-09T04:23:59.802914+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799178+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:23:59.799178+0000","last_peered":"2023-10-09T04:23:59.799178+0000","last_clean":"2023-10-09T04:23:59.799178+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:23:59.799178+0000","last_undegraded":"2023-10-09T04:23:59.799178+0000","last_fullsized":"2023-10-09T04:23:59.799178+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827521+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:24:00.827521+0000","last_peered":"2023-10-09T04:24:00.827521+0000","last_clean":"2023-10-09T04:24:00.827521+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:24:00.827521+0000","last_undegraded":"2023-10-09T04:24:00.827521+0000","last_fullsized":"2023-10-09T04:24:00.827521+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826876+0000","last_change":"2023-10-09T04:24:00.826876+0000","last_active":"2023-10-09T04:24:00.826876+0000","last_peered":"2023-10-09T04:24:00.826876+0000","last_clean":"2023-10-09T04:24:00.826876+0000","last_became_active":"2023-10-09T04:24:00.826469+0000","last_became_peered":"2023-10-09T04:24:00.826469+0000","last_unstale":"2023-10-09T04:24:00.826876+0000","last_undegraded":"2023-10-09T04:24:00.826876+0000","last_fullsized":"2023-10-09T04:24:00.826876+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:08.758269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.804987+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:24:00.804987+0000","last_peered":"2023-10-09T04:24:00.804987+0000","last_clean":"2023-10-09T04:24:00.804987+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:24:00.804987+0000","last_undegraded":"2023-10-09T04:24:00.804987+0000","last_fullsized":"2023-10-09T04:24:00.804987+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822324+0000","last_change":"2023-10-09T04:24:00.822324+0000","last_active":"2023-10-09T04:24:00.822324+0000","last_peered":"2023-10-09T04:24:00.822324+0000","last_clean":"2023-10-09T04:24:00.822324+0000","last_became_active":"2023-10-09T04:24:00.821777+0000","last_became_peered":"2023-10-09T04:24:00.821777+0000","last_unstale":"2023-10-09T04:24:00.822324+0000","last_undegraded":"2023-10-09T04:24:00.822324+0000","last_fullsized":"2023-10-09T04:24:00.822324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.798937+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:23:59.798937+0000","last_peered":"2023-10-09T04:23:59.798937+0000","last_clean":"2023-10-09T04:23:59.798937+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:23:59.798937+0000","last_undegraded":"2023-10-09T04:23:59.798937+0000","last_fullsized":"2023-10-09T04:23:59.798937+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797398+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:23:59.797398+0000","last_peered":"2023-10-09T04:23:59.797398+0000","last_clean":"2023-10-09T04:23:59.797398+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:23:59.797398+0000","last_undegraded":"2023-10-09T04:23:59.797398+0000","last_fullsized":"2023-10-09T04:23:59.797398+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.827553+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:24:00.827553+0000","last_peered":"2023-10-09T04:24:00.827553+0000","last_clean":"2023-10-09T04:24:00.827553+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:24:00.827553+0000","last_undegraded":"2023-10-09T04:24:00.827553+0000","last_fullsized":"2023-10-09T04:24:00.827553+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797418+0000","last_change":"2023-10-09T04:22:50.692658+0000","last_active":"2023-10-09T04:23:59.797418+0000","last_peered":"2023-10-09T04:23:59.797418+0000","last_clean":"2023-10-09T04:23:59.797418+0000","last_became_active":"2023-10-09T04:22:50.692027+0000","last_became_peered":"2023-10-09T04:22:50.692027+0000","last_unstale":"2023-10-09T04:23:59.797418+0000","last_undegraded":"2023-10-09T04:23:59.797418+0000","last_fullsized":"2023-10-09T04:23:59.797418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.792610+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:23:59.792610+0000","last_peered":"2023-10-09T04:23:59.792610+0000","last_clean":"2023-10-09T04:23:59.792610+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:23:59.792610+0000","last_undegraded":"2023-10-09T04:23:59.792610+0000","last_fullsized":"2023-10-09T04:23:59.792610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797868+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:23:59.797868+0000","last_peered":"2023-10-09T04:23:59.797868+0000","last_clean":"2023-10-09T04:23:59.797868+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:23:59.797868+0000","last_undegraded":"2023-10-09T04:23:59.797868+0000","last_fullsized":"2023-10-09T04:23:59.797868+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822512+0000","last_change":"2023-10-09T04:24:00.822512+0000","last_active":"2023-10-09T04:24:00.822512+0000","last_peered":"2023-10-09T04:24:00.822512+0000","last_clean":"2023-10-09T04:24:00.822512+0000","last_became_active":"2023-10-09T04:24:00.822183+0000","last_became_peered":"2023-10-09T04:24:00.822183+0000","last_unstale":"2023-10-09T04:24:00.822512+0000","last_undegraded":"2023-10-09T04:24:00.822512+0000","last_fullsized":"2023-10-09T04:24:00.822512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.029015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799467+0000","last_change":"2023-10-09T04:22:23.612613+0000","last_active":"2023-10-09T04:23:59.799467+0000","last_peered":"2023-10-09T04:23:59.799467+0000","last_clean":"2023-10-09T04:23:59.799467+0000","last_became_active":"2023-10-09T04:22:23.611940+0000","last_became_peered":"2023-10-09T04:22:23.611940+0000","last_unstale":"2023-10-09T04:23:59.799467+0000","last_undegraded":"2023-10-09T04:23:59.799467+0000","last_fullsized":"2023-10-09T04:23:59.799467+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822959+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:24:00.822959+0000","last_peered":"2023-10-09T04:24:00.822959+0000","last_clean":"2023-10-09T04:24:00.822959+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:24:00.822959+0000","last_undegraded":"2023-10-09T04:24:00.822959+0000","last_fullsized":"2023-10-09T04:24:00.822959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.796146+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:23:59.796146+0000","last_peered":"2023-10-09T04:23:59.796146+0000","last_clean":"2023-10-09T04:23:59.796146+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:23:59.796146+0000","last_undegraded":"2023-10-09T04:23:59.796146+0000","last_fullsized":"2023-10-09T04:23:59.796146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822638+0000","last_change":"2023-10-09T04:24:00.822638+0000","last_active":"2023-10-09T04:24:00.822638+0000","last_peered":"2023-10-09T04:24:00.822638+0000","last_clean":"2023-10-09T04:24:00.822638+0000","last_became_active":"2023-10-09T04:24:00.822103+0000","last_became_peered":"2023-10-09T04:24:00.822103+0000","last_unstale":"2023-10-09T04:24:00.822638+0000","last_undegraded":"2023-10-09T04:24:00.822638+0000","last_fullsized":"2023-10-09T04:24:00.822638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826616+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:24:00.826616+0000","last_peered":"2023-10-09T04:24:00.826616+0000","last_clean":"2023-10-09T04:24:00.826616+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:24:00.826616+0000","last_undegraded":"2023-10-09T04:24:00.826616+0000","last_fullsized":"2023-10-09T04:24:00.826616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.821914+0000","last_change":"2023-10-09T04:24:00.821914+0000","last_active":"2023-10-09T04:24:00.821914+0000","last_peered":"2023-10-09T04:24:00.821914+0000","last_clean":"2023-10-09T04:24:00.821914+0000","last_became_active":"2023-10-09T04:24:00.821588+0000","last_became_peered":"2023-10-09T04:24:00.821588+0000","last_unstale":"2023-10-09T04:24:00.821914+0000","last_undegraded":"2023-10-09T04:24:00.821914+0000","last_fullsized":"2023-10-09T04:24:00.821914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:52.558769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.797383+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:23:59.797383+0000","last_peered":"2023-10-09T04:23:59.797383+0000","last_clean":"2023-10-09T04:23:59.797383+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:23:59.797383+0000","last_undegraded":"2023-10-09T04:23:59.797383+0000","last_fullsized":"2023-10-09T04:23:59.797383+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802062+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:23:59.802062+0000","last_peered":"2023-10-09T04:23:59.802062+0000","last_clean":"2023-10-09T04:23:59.802062+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:23:59.802062+0000","last_undegraded":"2023-10-09T04:23:59.802062+0000","last_fullsized":"2023-10-09T04:23:59.802062+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.802932+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:23:59.802932+0000","last_peered":"2023-10-09T04:23:59.802932+0000","last_clean":"2023-10-09T04:23:59.802932+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:23:59.802932+0000","last_undegraded":"2023-10-09T04:23:59.802932+0000","last_fullsized":"2023-10-09T04:23:59.802932+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.805158+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:24:00.805158+0000","last_peered":"2023-10-09T04:24:00.805158+0000","last_clean":"2023-10-09T04:24:00.805158+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:24:00.805158+0000","last_undegraded":"2023-10-09T04:24:00.805158+0000","last_fullsized":"2023-10-09T04:24:00.805158+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.824188+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:24:00.824188+0000","last_peered":"2023-10-09T04:24:00.824188+0000","last_clean":"2023-10-09T04:24:00.824188+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:24:00.824188+0000","last_undegraded":"2023-10-09T04:24:00.824188+0000","last_fullsized":"2023-10-09T04:24:00.824188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822436+0000","last_change":"2023-10-09T04:24:00.822436+0000","last_active":"2023-10-09T04:24:00.822436+0000","last_peered":"2023-10-09T04:24:00.822436+0000","last_clean":"2023-10-09T04:24:00.822436+0000","last_became_active":"2023-10-09T04:24:00.822108+0000","last_became_peered":"2023-10-09T04:24:00.822108+0000","last_unstale":"2023-10-09T04:24:00.822436+0000","last_undegraded":"2023-10-09T04:24:00.822436+0000","last_fullsized":"2023-10-09T04:24:00.822436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:24.737042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.793754+0000","last_change":"2023-10-09T04:22:50.698679+0000","last_active":"2023-10-09T04:23:59.793754+0000","last_peered":"2023-10-09T04:23:59.793754+0000","last_clean":"2023-10-09T04:23:59.793754+0000","last_became_active":"2023-10-09T04:22:50.698360+0000","last_became_peered":"2023-10-09T04:22:50.698360+0000","last_unstale":"2023-10-09T04:23:59.793754+0000","last_undegraded":"2023-10-09T04:23:59.793754+0000","last_fullsized":"2023-10-09T04:23:59.793754+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.820037+0000","last_change":"2023-10-09T04:24:00.820037+0000","last_active":"2023-10-09T04:24:00.820037+0000","last_peered":"2023-10-09T04:24:00.820037+0000","last_clean":"2023-10-09T04:24:00.820037+0000","last_became_active":"2023-10-09T04:24:00.819650+0000","last_became_peered":"2023-10-09T04:24:00.819650+0000","last_unstale":"2023-10-09T04:24:00.820037+0000","last_undegraded":"2023-10-09T04:24:00.820037+0000","last_fullsized":"2023-10-09T04:24:00.820037+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:32.924825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.816756+0000","last_change":"2023-10-09T04:24:00.816214+0000","last_active":"2023-10-09T04:24:00.816756+0000","last_peered":"2023-10-09T04:24:00.816756+0000","last_clean":"2023-10-09T04:24:00.816756+0000","last_became_active":"2023-10-09T04:24:00.815867+0000","last_became_peered":"2023-10-09T04:24:00.815867+0000","last_unstale":"2023-10-09T04:24:00.816756+0000","last_undegraded":"2023-10-09T04:24:00.816756+0000","last_fullsized":"2023-10-09T04:24:00.816756+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:24.219649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.799344+0000","last_change":"2023-10-09T04:22:50.702787+0000","last_active":"2023-10-09T04:23:59.799344+0000","last_peered":"2023-10-09T04:23:59.799344+0000","last_clean":"2023-10-09T04:23:59.799344+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:22:50.702392+0000","last_unstale":"2023-10-09T04:23:59.799344+0000","last_undegraded":"2023-10-09T04:23:59.799344+0000","last_fullsized":"2023-10-09T04:23:59.799344+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.814141+0000","last_change":"2023-10-09T04:24:00.814141+0000","last_active":"2023-10-09T04:24:00.814141+0000","last_peered":"2023-10-09T04:24:00.814141+0000","last_clean":"2023-10-09T04:24:00.814141+0000","last_became_active":"2023-10-09T04:24:00.813799+0000","last_became_peered":"2023-10-09T04:24:00.813799+0000","last_unstale":"2023-10-09T04:24:00.814141+0000","last_undegraded":"2023-10-09T04:24:00.814141+0000","last_fullsized":"2023-10-09T04:24:00.814141+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:37.871798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.822980+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:24:00.822980+0000","last_peered":"2023-10-09T04:24:00.822980+0000","last_clean":"2023-10-09T04:24:00.822980+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:24:00.822980+0000","last_undegraded":"2023-10-09T04:24:00.822980+0000","last_fullsized":"2023-10-09T04:24:00.822980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826025+0000","last_change":"2023-10-09T04:24:00.825647+0000","last_active":"2023-10-09T04:24:00.826025+0000","last_peered":"2023-10-09T04:24:00.826025+0000","last_clean":"2023-10-09T04:24:00.826025+0000","last_became_active":"2023-10-09T04:24:00.825310+0000","last_became_peered":"2023-10-09T04:24:00.825310+0000","last_unstale":"2023-10-09T04:24:00.826025+0000","last_undegraded":"2023-10-09T04:24:00.826025+0000","last_fullsized":"2023-10-09T04:24:00.826025+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:07.180568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.826023+0000","last_change":"2023-10-09T04:24:00.826023+0000","last_active":"2023-10-09T04:24:00.826023+0000","last_peered":"2023-10-09T04:24:00.826023+0000","last_clean":"2023-10-09T04:24:00.826023+0000","last_became_active":"2023-10-09T04:24:00.825466+0000","last_became_peered":"2023-10-09T04:24:00.825466+0000","last_unstale":"2023-10-09T04:24:00.826023+0000","last_undegraded":"2023-10-09T04:24:00.826023+0000","last_fullsized":"2023-10-09T04:24:00.826023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:39.932991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.800944+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:23:59.800944+0000","last_peered":"2023-10-09T04:23:59.800944+0000","last_clean":"2023-10-09T04:23:59.800944+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:23:59.800944+0000","last_undegraded":"2023-10-09T04:23:59.800944+0000","last_fullsized":"2023-10-09T04:23:59.800944+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430421,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5365985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.56499999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92400000000000004,"15min":0.91600000000000004},"min":{"1min":0.624,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.212,"5min":1.375,"15min":1.375},"last":1.0940000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.92000000000000004,"15min":0.92700000000000005},"min":{"1min":0.59099999999999997,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.1719999999999999,"5min":2.097,"15min":2.097},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.94499999999999995,"15min":0.93899999999999995},"min":{"1min":0.625,"5min":0.503,"15min":0.503},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.98699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.91600000000000004,"15min":0.92200000000000004},"min":{"1min":0.69499999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1040000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.98399999999999999,"15min":0.98899999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.242,"15min":1.242},"last":0.82999999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.026,"15min":1.026},"min":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.05,"15min":1.042},"min":{"1min":0.79800000000000004,"5min":0.749,"15min":0.749},"max":{"1min":1.288,"5min":1.325,"15min":1.325},"last":1.054}]},{"osd":4,"last update":"Mon Oct 9 04:26:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.92400000000000004,"15min":0.91700000000000004},"min":{"1min":0.63400000000000001,"5min":0.55900000000000005,"15min":0.52900000000000003},"max":{"1min":1.353,"5min":1.4550000000000001,"15min":2.0419999999999998},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.98399999999999999,"15min":0.97099999999999997},"min":{"1min":0.69599999999999995,"5min":0.58499999999999996,"15min":0.495},"max":{"1min":1.29,"5min":1.4399999999999999,"15min":2.1139999999999999},"last":0.96299999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:26:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97199999999999998,"15min":0.99399999999999999},"min":{"1min":0.66900000000000004,"5min":0.51500000000000001,"15min":0.497},"max":{"1min":1.375,"5min":1.375,"15min":1.9510000000000001},"last":0.997},{"interface":"front","average":{"1min":1.004,"5min":1.0329999999999999,"15min":1.0269999999999999},"min":{"1min":0.76000000000000001,"5min":0.66500000000000004,"15min":0.45500000000000002},"max":{"1min":1.409,"5min":1.409,"15min":2.2349999999999999},"last":0.877}]},{"osd":6,"last update":"Mon Oct 9 04:26:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0640000000000001,"15min":1.0629999999999999},"min":{"1min":0.71999999999999997,"5min":0.57099999999999995,"15min":0.46100000000000002},"max":{"1min":1.46,"5min":1.5,"15min":2.3100000000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.056,"15min":1.0529999999999999},"min":{"1min":0.85699999999999998,"5min":0.53900000000000003,"15min":0.48199999999999998},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":2.1909999999999998},"last":0.91500000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626654,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298728,"kb_used_data":1632,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446424,"statfs":{"total":95995035648,"available":95689138176,"internally_reserved":0,"allocated":1671168,"data_stored":1171554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.88700000000000001,"15min":0.89100000000000001},"min":{"1min":0.57299999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2270000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.0629999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.88600000000000001,"15min":0.89200000000000002},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1619999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.1759999999999999},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91200000000000003,"15min":0.90200000000000002},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.1419999999999999,"5min":1.587,"15min":1.587},"last":0.98299999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.95399999999999996,"15min":0.96499999999999997},"min":{"1min":0.64000000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.204,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":1.1299999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.2250000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.0029999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:26:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.91400000000000003,"15min":0.91600000000000004},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.159,"5min":1.256,"15min":1.256},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.05,"5min":1.042,"15min":1.04},"min":{"1min":0.85999999999999999,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.90300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.93700000000000006,"15min":0.91700000000000004},"min":{"1min":0.56799999999999995,"5min":0.51800000000000002,"15min":0.40899999999999997},"max":{"1min":1.147,"5min":1.367,"15min":1.663},"last":1.089},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.99199999999999999,"15min":0.95799999999999996},"min":{"1min":0.65900000000000003,"5min":0.51200000000000001,"15min":0.443},"max":{"1min":1.319,"5min":1.405,"15min":1.548},"last":1.1950000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.0149999999999999,"15min":1.0369999999999999},"min":{"1min":0.67700000000000005,"5min":0.55900000000000005,"15min":0.50700000000000001},"max":{"1min":1.3009999999999999,"5min":1.498,"15min":3.9180000000000001},"last":1.2150000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0289999999999999,"15min":1.0049999999999999},"min":{"1min":0.745,"5min":0.56100000000000005,"15min":0.505},"max":{"1min":1.278,"5min":1.3320000000000001,"15min":1.635},"last":1.151}]},{"osd":7,"last update":"Mon Oct 9 04:25:13 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0860000000000001,"15min":1.0720000000000001},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.626},"max":{"1min":1.2609999999999999,"5min":1.5569999999999999,"15min":1.9570000000000001},"last":1.103},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1679999999999999,"15min":1.159},"min":{"1min":0.85399999999999998,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3149999999999999,"5min":1.6279999999999999,"15min":1.9990000000000001},"last":1.304}]}]},{"osd":1,"up_from":93,"seq":399431958570,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":6,"kb_used_meta":31545,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":585588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":32302092},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.66600000000000004,"15min":0.67100000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.90500000000000003,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.72299999999999998,"15min":0.73899999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.93200000000000005,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.64600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:25:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.66400000000000003,"15min":0.67300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.76700000000000002,"15min":0.77900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.234,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.88600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:26:04 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.82899999999999996,"15min":0.82699999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.036},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.68600000000000005,"15min":0.68500000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.91200000000000003,"15min":0.91800000000000004},"min":{"1min":0.61199999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1759999999999999,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.024,"15min":1.0389999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.333,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.1299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.99199999999999999,"15min":1.01},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.4550000000000001,"5min":1.819,"15min":1.819},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.97899999999999998,"15min":0.98499999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3009999999999999,"5min":1.595,"15min":1.595},"last":1.117}]},{"osd":6,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.016,"15min":1.026},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.5449999999999999,"5min":1.877,"15min":1.877},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.0009999999999999,"15min":1.0169999999999999},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2769999999999999,"5min":1.724,"15min":1.724},"last":1.0840000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0029999999999999,"15min":1.004},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.349,"5min":1.653,"15min":1.653},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0920000000000001,"15min":1.1120000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.4019999999999999,"5min":1.786,"15min":1.786},"last":1.3160000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122096,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33304,"kb_used_data":1680,"kb_used_omap":9,"kb_used_meta":31606,"kb_avail":93711848,"statfs":{"total":95995035648,"available":95960932352,"internally_reserved":0,"allocated":1720320,"data_stored":1178304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":32365041},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:25:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.63300000000000001,"15min":0.64100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.64800000000000002,"15min":0.66400000000000003},"min":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80700000000000005,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.75700000000000001,"15min":0.77500000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":1.008}]},{"osd":3,"last update":"Mon Oct 9 04:26:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.82699999999999996,"15min":0.82599999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.874}]},{"osd":4,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0249999999999999,"15min":1.0720000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1180000000000001,"5min":2.036,"15min":2.036},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.0209999999999999,"15min":1.0349999999999999},"min":{"1min":0.69499999999999995,"5min":0.63,"15min":0.63},"max":{"1min":1.25,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.1619999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96799999999999997,"15min":0.96499999999999997},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.302,"5min":1.7829999999999999,"15min":1.7829999999999999},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.99299999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1379999999999999,"5min":1.613,"15min":1.613},"last":1.111}]},{"osd":6,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1,"15min":1.028},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.2769999999999999,"5min":1.712,"15min":1.712},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.012,"15min":1.012},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2869999999999999,"5min":1.641,"15min":1.641},"last":1.296}]},{"osd":7,"last update":"Mon Oct 9 04:25:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.093,"15min":1.123},"min":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.2649999999999999,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":1.262},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.101,"15min":1.137},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1799999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":1.1359999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795045,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36728,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31540,"kb_avail":93708424,"statfs":{"total":95995035648,"available":95957426176,"internally_reserved":0,"allocated":5292032,"data_stored":4777245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":32297002},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.66700000000000004,"15min":0.69099999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.80400000000000005,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.33300000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.91000000000000003,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.72399999999999998,"15min":0.73599999999999999},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.70999999999999996,"15min":0.71199999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.79600000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:26:06 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.81299999999999994,"15min":0.82199999999999995},"min":{"1min":0.51100000000000001,"5min":0.501,"15min":0.501},"max":{"1min":0.97899999999999998,"5min":2.2759999999999998,"15min":2.2759999999999998},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.84699999999999998,"15min":0.84599999999999997},"min":{"1min":0.54500000000000004,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.133,"5min":2.1899999999999999,"15min":2.1899999999999999},"last":0.93799999999999994}]},{"osd":4,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94299999999999995,"15min":0.94599999999999995},"min":{"1min":0.66100000000000003,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.083,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.075},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.94399999999999995,"15min":0.95799999999999996},"min":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.1579999999999999,"5min":1.429,"15min":1.429},"last":0.89600000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.024},"min":{"1min":0.70099999999999996,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2989999999999999,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":1.127},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.262,"5min":1.718,"15min":1.718},"last":0.81699999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.006,"15min":1.02},"min":{"1min":0.60399999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3440000000000001,"5min":1.855,"15min":1.855},"last":0.748},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.05,"15min":1.0649999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2769999999999999,"5min":1.603,"15min":1.603},"last":0.91300000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.115,"15min":1.139},"min":{"1min":0.63900000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3480000000000001,"5min":1.825,"15min":1.825},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.995,"5min":1.077,"15min":1.091},"min":{"1min":0.67900000000000005,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.2949999999999999,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":1.171}]}]},{"osd":3,"up_from":103,"seq":442381631516,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32552,"kb_used_data":1056,"kb_used_omap":5,"kb_used_meta":31482,"kb_avail":93712600,"statfs":{"total":95995035648,"available":95961702400,"internally_reserved":0,"allocated":1081344,"data_stored":580896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":32237742},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.74099999999999999,"15min":0.76900000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.97799999999999998,"5min":1.615,"15min":1.615},"last":0.751},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.68200000000000005,"15min":0.68899999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.82899999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.69499999999999995,"15min":0.71399999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.93100000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.79800000000000004,"15min":0.82799999999999996},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.93100000000000005,"5min":2.145,"15min":2.145},"last":1.0469999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.78600000000000003,"15min":0.80800000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.96599999999999997,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.91600000000000004,"15min":0.95799999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.099,"5min":2.23,"15min":2.23},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.0509999999999999,"15min":1.083},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.131,"5min":2.0470000000000002,"15min":2.0470000000000002},"last":1.2330000000000001},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.97999999999999998,"15min":0.995},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.345,"5min":1.345,"15min":1.345},"last":0.95899999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.026,"15min":1.044},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2729999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.0509999999999999,"15min":1.0740000000000001},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.4359999999999999,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.1870000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.0309999999999999,"15min":1.0489999999999999},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.391,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":1.008},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0780000000000001,"15min":1.099},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.343,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.2869999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.119,"15min":1.1359999999999999},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.413},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.1930000000000001,"15min":1.2270000000000001},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.292,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":1.3220000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019118,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84699999999999998,"15min":0.85699999999999998},"min":{"1min":0.54000000000000004,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.165,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.161},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.88200000000000001,"15min":0.90000000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0629999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.73799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.88700000000000001,"15min":0.90400000000000003},"min":{"1min":0.54900000000000004,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1439999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0269999999999999,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.98399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.629,"5min":0.502,"15min":0.502},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.96099999999999997,"15min":0.95999999999999996},"min":{"1min":0.64100000000000001,"5min":0.623,"15min":0.623},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.80900000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.0309999999999999,"15min":1.034},"min":{"1min":0.83299999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3009999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0720000000000001,"15min":1.0820000000000001},"min":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3300000000000001,"5min":1.363,"15min":1.363},"last":1.242}]},{"osd":5,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.95499999999999996,"15min":0.93999999999999995},"min":{"1min":0.59999999999999998,"5min":0.56399999999999995,"15min":0.433},"max":{"1min":1.266,"5min":1.5269999999999999,"15min":1.5509999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.93700000000000006,"15min":0.90500000000000003},"min":{"1min":0.69999999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.1970000000000001,"5min":1.877,"15min":1.877},"last":1.0489999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:28 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.99099999999999999,"15min":0.96299999999999997},"min":{"1min":0.755,"5min":0.56100000000000005,"15min":0.47099999999999997},"max":{"1min":1.389,"5min":1.389,"15min":1.7290000000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.02,"5min":1.0549999999999999,"15min":1.026},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.46200000000000002},"max":{"1min":1.3740000000000001,"5min":1.583,"15min":1.796},"last":1.3149999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.157,"15min":1.125},"min":{"1min":0.82999999999999996,"5min":0.82599999999999996,"15min":0.65900000000000003},"max":{"1min":1.5409999999999999,"5min":1.653,"15min":1.8600000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1319999999999999,"15min":1.1080000000000001},"min":{"1min":0.88800000000000001,"5min":0.76500000000000001,"15min":0.57699999999999996},"max":{"1min":1.3819999999999999,"5min":1.512,"15min":1.923},"last":1.131}]}]},{"osd":5,"up_from":36,"seq":154618822885,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302232,"kb_used_data":5136,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93442920,"statfs":{"total":95995035648,"available":95685550080,"internally_reserved":0,"allocated":5259264,"data_stored":4770995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.87,"15min":0.872},"min":{"1min":0.63600000000000001,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.1399999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.74399999999999999,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.107}]},{"osd":1,"last update":"Mon Oct 9 04:25:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.97399999999999998,"15min":0.995},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2549999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.022,"5min":0.96399999999999997,"15min":0.95399999999999996},"min":{"1min":0.71099999999999997,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.4019999999999999,"5min":1.645,"15min":1.645},"last":0.80000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:25:23 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93100000000000005,"15min":0.93600000000000005},"min":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.871},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.0189999999999999,"15min":1.0209999999999999},"min":{"1min":0.79700000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.266,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.65400000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.996,"15min":0.98999999999999999},"min":{"1min":0.82499999999999996,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0569999999999999,"15min":1.052},"min":{"1min":0.90300000000000002,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.276,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.76500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:42 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.93600000000000005,"15min":0.94699999999999995},"min":{"1min":0.70999999999999996,"5min":0.51600000000000001,"15min":0.47099999999999997},"max":{"1min":1.383,"5min":1.383,"15min":2.165},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.075,"5min":1.0429999999999999,"15min":0.997},"min":{"1min":0.72599999999999998,"5min":0.54600000000000004,"15min":0.47599999999999998},"max":{"1min":1.4319999999999999,"5min":3.7989999999999999,"15min":3.7989999999999999},"last":1.0149999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0229999999999999,"15min":1.0069999999999999},"min":{"1min":0.77000000000000002,"5min":0.50700000000000001,"15min":0.40899999999999997},"max":{"1min":1.5700000000000001,"5min":2.2999999999999998,"15min":2.2999999999999998},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.083,"15min":1.0720000000000001},"min":{"1min":0.68500000000000005,"5min":0.58799999999999997,"15min":0.44800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":2.5840000000000001},"last":0.96999999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.1359999999999999,"15min":1.1140000000000001},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.42599999999999999},"max":{"1min":1.5589999999999999,"5min":1.6160000000000001,"15min":2.7050000000000001},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.089,"15min":1.1020000000000001},"min":{"1min":0.747,"5min":0.73999999999999999,"15min":0.47799999999999998},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":2.2829999999999999},"last":1.073}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:26:14.739 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:26:14.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T04:26:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:15 smithi153 bash[59565]: audit 2023-10-09T04:26:13.943270+0000 mgr.y (mgr.34107) 354 : audit 0 from='client.44645 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:15 smithi153 bash[59565]: audit 2023-10-09T04:26:15.367481+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:15 smithi082 bash[77063]: audit 2023-10-09T04:26:13.943270+0000 mgr.y (mgr.34107) 354 : audit 0 from='client.44645 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:15.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:15 smithi082 bash[77063]: audit 2023-10-09T04:26:15.367481+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:15 smithi082 bash[79733]: audit 2023-10-09T04:26:13.943270+0000 mgr.y (mgr.34107) 354 : audit 0 from='client.44645 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:26:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:15 smithi082 bash[79733]: audit 2023-10-09T04:26:15.367481+0000 mon.a (mon.0) 520 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:16 smithi153 bash[59565]: cluster 2023-10-09T04:26:15.329771+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:16.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:16 smithi082 bash[79733]: cluster 2023-10-09T04:26:15.329771+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:16.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:16 smithi082 bash[77063]: cluster 2023-10-09T04:26:15.329771+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:18.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:18 smithi153 bash[59565]: cluster 2023-10-09T04:26:17.330498+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:18.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:18 smithi082 bash[77063]: cluster 2023-10-09T04:26:17.330498+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:18.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:18 smithi082 bash[79733]: cluster 2023-10-09T04:26:17.330498+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:19.552 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:19] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.33.4" 2023-10-09T04:26:20.622 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:20 smithi082 bash[77063]: cluster 2023-10-09T04:26:19.331273+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:20.622 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:20 smithi082 bash[79733]: cluster 2023-10-09T04:26:19.331273+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:20 smithi153 bash[59565]: cluster 2023-10-09T04:26:19.331273+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:21 smithi153 bash[59565]: audit 2023-10-09T04:26:20.763585+0000 mon.a (mon.0) 521 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:21 smithi153 bash[59565]: audit 2023-10-09T04:26:20.772073+0000 mon.a (mon.0) 522 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:21 smithi153 bash[59565]: cluster 2023-10-09T04:26:21.332086+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:21 smithi153 bash[59565]: audit 2023-10-09T04:26:21.668092+0000 mon.a (mon.0) 523 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:21 smithi153 bash[59565]: audit 2023-10-09T04:26:21.677079+0000 mon.a (mon.0) 524 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[79733]: audit 2023-10-09T04:26:20.763585+0000 mon.a (mon.0) 521 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[79733]: audit 2023-10-09T04:26:20.772073+0000 mon.a (mon.0) 522 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[79733]: cluster 2023-10-09T04:26:21.332086+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:22.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[79733]: audit 2023-10-09T04:26:21.668092+0000 mon.a (mon.0) 523 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[79733]: audit 2023-10-09T04:26:21.677079+0000 mon.a (mon.0) 524 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[77063]: audit 2023-10-09T04:26:20.763585+0000 mon.a (mon.0) 521 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[77063]: audit 2023-10-09T04:26:20.772073+0000 mon.a (mon.0) 522 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[77063]: cluster 2023-10-09T04:26:21.332086+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[77063]: audit 2023-10-09T04:26:21.668092+0000 mon.a (mon.0) 523 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:21 smithi082 bash[77063]: audit 2023-10-09T04:26:21.677079+0000 mon.a (mon.0) 524 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:22.693 INFO:teuthology.orchestra.run.smithi082.stdout:epoch 104 2023-10-09T04:26:22.693 INFO:teuthology.orchestra.run.smithi082.stdout:fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:26:22.693 INFO:teuthology.orchestra.run.smithi082.stdout:created 2023-10-09T04:00:57.352609+0000 2023-10-09T04:26:22.693 INFO:teuthology.orchestra.run.smithi082.stdout:modified 2023-10-09T04:24:00.790711+0000 2023-10-09T04:26:22.693 INFO:teuthology.orchestra.run.smithi082.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:crush_version 22 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:full_ratio 0.95 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:backfillfull_ratio 0.9 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:nearfull_ratio 0.85 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:require_min_compat_client luminous 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:min_compat_client luminous 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:require_osd_release pacific 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:stretch_mode_enabled false 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 82 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T04:26:22.694 INFO:teuthology.orchestra.run.smithi082.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:max_osd 8 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288] [v2:172.21.15.82:6804/85623288,v1:172.21.15.82:6805/85623288] exists,up a068a4f9-387d-4d69-afa5-d160994babe9 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898] [v2:172.21.15.82:6812/1232825898,v1:172.21.15.82:6813/1232825898] exists,up fcfd2467-bc5b-4cf9-8fbb-71ee2b847070 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682] [v2:172.21.15.82:6820/2407238682,v1:172.21.15.82:6821/2407238682] exists,up 0b236da8-a93d-4c97-a234-b9033c7c7067 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592] [v2:172.21.15.82:6828/3740620592,v1:172.21.15.82:6829/3740620592] exists,up 0ae42ef8-7a8b-4eec-927b-01af15ae54f1 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063] [v2:172.21.15.153:6802/2316881063,v1:172.21.15.153:6803/2316881063] exists,up 298364f7-1ee4-436d-81d7-1ddf54d45607 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071] [v2:172.21.15.153:6810/1288836071,v1:172.21.15.153:6811/1288836071] exists,up 4d79acac-f278-473a-8dbf-9cf495c9f498 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505] [v2:172.21.15.153:6818/2167557505,v1:172.21.15.153:6819/2167557505] exists,up b3727fa4-1f3c-4a8e-89b4-a9e631638f17 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403] [v2:172.21.15.153:6826/3367585403,v1:172.21.15.153:6827/3367585403] exists,up 2ae5ef44-7804-4aa6-9343-0cd93c170de2 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:pg_upmap_items 4.b [1,2] 2023-10-09T04:26:22.695 INFO:teuthology.orchestra.run.smithi082.stdout:pg_upmap_items 4.1c [1,5] 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:pg_upmap_items 4.1d [1,2] 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/3203316964 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/2145280144 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/3596291160 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6801/1028221221 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6801/2283823309 expires 2023-10-10T04:01:24.719169+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6800/2283823309 expires 2023-10-10T04:01:24.719169+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/1174531512 expires 2023-10-10T04:09:06.794971+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/2959459015 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:0/1415425338 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6800/1028221221 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.696 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6801/886924328 expires 2023-10-10T04:01:54.491441+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6801/3756706366 expires 2023-10-10T04:09:06.794971+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/2018378994 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/2478756274 expires 2023-10-10T04:09:06.794971+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6800/1402928284 expires 2023-10-10T04:14:52.640519+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6800/886924328 expires 2023-10-10T04:01:54.491441+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:6835/4277057295 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/838245020 expires 2023-10-10T04:01:24.719169+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:6834/4277057295 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/1663762755 expires 2023-10-10T04:01:54.491441+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:0/1290610280 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/3909792950 expires 2023-10-10T04:17:29.105174+0000 2023-10-09T04:26:22.697 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6800/3756706366 expires 2023-10-10T04:09:06.794971+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/1703800946 expires 2023-10-10T04:01:54.491441+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/715502811 expires 2023-10-10T04:01:24.719169+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:0/1754741933 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.153:0/3564317560 expires 2023-10-10T04:16:27.265506+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/590497495 expires 2023-10-10T04:14:52.640519+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:6801/1402928284 expires 2023-10-10T04:14:52.640519+0000 2023-10-09T04:26:22.698 INFO:teuthology.orchestra.run.smithi082.stdout:blocklist 172.21.15.82:0/3169418397 expires 2023-10-10T04:14:52.640519+0000 2023-10-09T04:26:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:22 smithi153 bash[59565]: audit 2023-10-09T04:26:22.147847+0000 mon.a (mon.0) 525 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:22 smithi153 bash[59565]: audit 2023-10-09T04:26:22.149173+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:22 smithi153 bash[59565]: audit 2023-10-09T04:26:22.157644+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:22 smithi153 bash[59565]: audit 2023-10-09T04:26:22.690510+0000 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.82:0/3295563461' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:26:23.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[79733]: audit 2023-10-09T04:26:22.147847+0000 mon.a (mon.0) 525 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:23.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[79733]: audit 2023-10-09T04:26:22.149173+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[79733]: audit 2023-10-09T04:26:22.157644+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[79733]: audit 2023-10-09T04:26:22.690510+0000 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.82:0/3295563461' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[77063]: audit 2023-10-09T04:26:22.147847+0000 mon.a (mon.0) 525 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[77063]: audit 2023-10-09T04:26:22.149173+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[77063]: audit 2023-10-09T04:26:22.157644+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:26:23.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:22 smithi082 bash[77063]: audit 2023-10-09T04:26:22.690510+0000 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.82:0/3295563461' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:26:23.532 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.82:6802/85623288,v1:172.21.15.82:6803/85623288] [v2:172.21.15.82:6804/85623288,v1:172.21.15.82:6805/85623288] exists,up a068a4f9-387d-4d69-afa5-d160994babe9', 'osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.82:6810/1232825898,v1:172.21.15.82:6811/1232825898] [v2:172.21.15.82:6812/1232825898,v1:172.21.15.82:6813/1232825898] exists,up fcfd2467-bc5b-4cf9-8fbb-71ee2b847070', 'osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.82:6818/2407238682,v1:172.21.15.82:6819/2407238682] [v2:172.21.15.82:6820/2407238682,v1:172.21.15.82:6821/2407238682] exists,up 0b236da8-a93d-4c97-a234-b9033c7c7067', 'osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.82:6826/3740620592,v1:172.21.15.82:6827/3740620592] [v2:172.21.15.82:6828/3740620592,v1:172.21.15.82:6829/3740620592] exists,up 0ae42ef8-7a8b-4eec-927b-01af15ae54f1', 'osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6800/2316881063,v1:172.21.15.153:6801/2316881063] [v2:172.21.15.153:6802/2316881063,v1:172.21.15.153:6803/2316881063] exists,up 298364f7-1ee4-436d-81d7-1ddf54d45607', 'osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6808/1288836071,v1:172.21.15.153:6809/1288836071] [v2:172.21.15.153:6810/1288836071,v1:172.21.15.153:6811/1288836071] exists,up 4d79acac-f278-473a-8dbf-9cf495c9f498', 'osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6816/2167557505,v1:172.21.15.153:6817/2167557505] [v2:172.21.15.153:6818/2167557505,v1:172.21.15.153:6819/2167557505] exists,up b3727fa4-1f3c-4a8e-89b4-a9e631638f17', 'osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.153:6824/3367585403,v1:172.21.15.153:6825/3367585403] [v2:172.21.15.153:6826/3367585403,v1:172.21.15.153:6827/3367585403] exists,up 2ae5ef44-7804-4aa6-9343-0cd93c170de2'] 2023-10-09T04:26:23.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T04:26:24.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:23 smithi153 bash[59565]: cluster 2023-10-09T04:26:23.332816+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:24.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:23 smithi082 bash[77063]: cluster 2023-10-09T04:26:23.332816+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:24.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:23 smithi082 bash[79733]: cluster 2023-10-09T04:26:23.332816+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:26 smithi153 bash[59565]: cluster 2023-10-09T04:26:25.333792+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:26 smithi082 bash[77063]: cluster 2023-10-09T04:26:25.333792+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:26 smithi082 bash[79733]: cluster 2023-10-09T04:26:25.333792+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:28.314 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:28.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:28 smithi153 bash[59565]: cluster 2023-10-09T04:26:27.334591+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:28.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:28 smithi082 bash[77063]: cluster 2023-10-09T04:26:27.334591+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:28.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:28 smithi082 bash[79733]: cluster 2023-10-09T04:26:27.334591+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:29.566 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:29 smithi082 bash[79733]: audit 2023-10-09T04:26:29.255228+0000 mon.a (mon.0) 529 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.567 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:29 smithi082 bash[77063]: audit 2023-10-09T04:26:29.255228+0000 mon.a (mon.0) 529 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:29 smithi153 bash[59565]: audit 2023-10-09T04:26:29.255228+0000 mon.a (mon.0) 529 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:29] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.33.4" 2023-10-09T04:26:30.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:30 smithi082 bash[79733]: cluster 2023-10-09T04:26:29.335204+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:30 smithi082 bash[77063]: cluster 2023-10-09T04:26:29.335204+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:30.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:30 smithi153 bash[59565]: cluster 2023-10-09T04:26:29.335204+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:31.322 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "name": "mon.a", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "cluster": "ceph", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "admin_socket_mode": "", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_client_required": "cephx, none", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_cluster_required": "cephx", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_debug": "true", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T04:26:31.323 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_service_required": "cephx", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "auth_supported": "", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_aio": "true", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_aio_reap_max": "16", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_async_discard": "false", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_block_size": "4096", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_debug_aio": "false", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T04:26:31.324 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_enable_discard": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_flock_retry": "3", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_inject_crash": "0", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_ioring": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_ioring_hipri": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T04:26:31.325 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bdev_type": "", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_allocator": "hybrid", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_buffered_io": "true", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T04:26:31.326 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_replay_recovery": "false", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluefs_sync_write": "false", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T04:26:31.327 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_allocator": "hybrid", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_create": "true", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_db_create": "false", 2023-10-09T04:26:31.328 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_db_path": "", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_db_size": "0", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_path": "", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_size": "107374182400", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_wal_create": "false", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_wal_path": "", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bluefs": "true", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T04:26:31.329 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_autotune": "true", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_size": "0", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T04:26:31.330 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cache_type": "2q", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_clone_cow": "true", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T04:26:31.331 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_mode": "none", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_freelist": "false", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T04:26:31.332 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T04:26:31.333 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fail_eio": "false", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T04:26:31.334 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_blob_size": "0", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T04:26:31.336 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T04:26:31.337 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_spdk_mem": "512", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T04:26:31.338 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_tracing": "false", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T04:26:31.339 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T04:26:31.340 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_cluster_require_version": "2", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_require_signatures": "false", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_require_version": "2", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_service_require_signatures": "false", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_service_require_version": "2", 2023-10-09T04:26:31.341 INFO:teuthology.orchestra.run.smithi082.stdout: "cephx_sign_messages": "true", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "chdir": "", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_acl_type": "", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_asio_thread_count": "2", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_cache_mid": "0.750000", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_cache_size": "16384", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_caps_release_delay": "5", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_check_pool_perm": "true", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_debug_force_sync_read": "false", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_debug_getattr_caps": "false", 2023-10-09T04:26:31.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_die_on_failed_remount": "false", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_dirsize_rbytes": "true", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_force_lazyio": "false", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_fs": "", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_inject_release_failure": "false", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_max_inline_size": "4096", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_mds_namespace": "", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_metadata": "", 2023-10-09T04:26:31.343 INFO:teuthology.orchestra.run.smithi082.stdout: "client_mount_gid": "-1", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_mount_timeout": "300", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_mount_uid": "-1", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_mountpoint": "/", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_notify_timeout": "10", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc": "true", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc_max_objects": "1000", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc_size": "209715200", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_permissions": "true", 2023-10-09T04:26:31.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_quota": "true", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_quota_df": "true", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_readahead_max_bytes": "0", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_readahead_max_periods": "4", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_readahead_min": "131072", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_reconnect_stale": "false", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_shutdown_timeout": "30", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_snapdir": ".snap", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_tick_interval": "1", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_trace": "", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_use_faked_inos": "false", 2023-10-09T04:26:31.345 INFO:teuthology.orchestra.run.smithi082.stdout: "client_use_random_mds": "false", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_graylog": "false", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_graylog_port": "12201", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_monitors": "default=true", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_syslog": "false", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "clog_to_syslog_level": "info", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "cluster_addr": "-", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "cluster_network": "", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "cluster_network_interface": "", 2023-10-09T04:26:31.346 INFO:teuthology.orchestra.run.smithi082.stdout: "compressor_zlib_isal": "false", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "compressor_zlib_level": "5", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "compressor_zstd_level": "1", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crimson_seastar_smp": "1", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crush_location": "", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crush_location_hook": "", 2023-10-09T04:26:31.347 INFO:teuthology.orchestra.run.smithi082.stdout: "crush_location_hook_timeout": "10", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "daemonize": "false", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "daos_pool": "tank", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_alienstore": "0/5", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_asok": "1/5", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_asok_assert_abort": "false", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_auth": "1/5", 2023-10-09T04:26:31.348 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_bdev": "1/3", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_bluefs": "1/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_bluestore": "1/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_buffer": "0/1", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_cephsqlite": "0/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_client": "0/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_compressor": "1/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_context": "0/1", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_crush": "1/1", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_crypto": "1/5", 2023-10-09T04:26:31.349 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_cyanstore": "0/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_dpdk": "1/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_eventtrace": "1/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_filer": "0/1", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_filestore": "1/3", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_finisher": "1/1", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_fuse": "1/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T04:26:31.350 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_javaclient": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_journal": "1/3", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_journaler": "0/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_kstore": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_leveldb": "4/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_lockdep": "0/1", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mclock": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds_balancer": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds_locker": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds_log": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T04:26:31.351 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mds_migrator": "1/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_memstore": "1/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mgr": "2/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mgrc": "1/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_mon": "20/20", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_monc": "0/10", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_ms": "1/1", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_none": "0/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_objclass": "0/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_objectcacher": "0/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_objecter": "0/1", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_optracker": "0/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_osd": "1/5", 2023-10-09T04:26:31.352 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_paxos": "20/20", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_perfcounter": "1/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_prioritycache": "1/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rados": "0/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rbd": "0/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rbd_replay": "0/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_refs": "0/0", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_reserver": "1/1", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw": "1/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw_access": "1/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T04:26:31.353 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw_flight": "1/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rgw_sync": "1/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_rocksdb": "4/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_backref": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_cache": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_device": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_epm": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T04:26:31.354 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_journal": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_lba": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_odata": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_omap": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_onode": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_t": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_seastore_tm": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_striper": "0/1", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_test": "0/5", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_throttle": "1/1", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_timer": "0/1", 2023-10-09T04:26:31.355 INFO:teuthology.orchestra.run.smithi082.stdout: "debug_tp": "0/5", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "device_failure_prediction_mode": "none", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "err_to_graylog": "false", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "err_to_journald": "false", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "err_to_stderr": "true", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "err_to_syslog": "false", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event_tracing": "false", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_http_port": "9926", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_prio_limit": "5", 2023-10-09T04:26:31.356 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_sort_metrics": "true", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "exporter_stats_period": "5", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "fake_statfs_for_testing": "0", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "fatal_signal_handlers": "true", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filer_max_purge_ops": "10", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filer_max_truncate_ops": "128", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_blackhole": "false", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_btrfs_snap": "true", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_caller_concurrency": "10", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T04:26:31.357 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_debug_omap_check": "false", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_debug_verify_split": "false", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_dump_file": "", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fadvise": "true", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fail_eio": "true", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T04:26:31.358 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fd_cache_size": "128", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fiemap": "false", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_inject_stall": "0", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_journal_parallel": "false", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_journal_trailing": "false", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_journal_writeahead": "false", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_kill_at": "0", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T04:26:31.359 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T04:26:31.360 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_merge_threshold": "-10", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_odsync_write": "false", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_omap_backend_path": "", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_op_threads": "2", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_punch_hole": "false", 2023-10-09T04:26:31.361 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_queue_max_ops": "50", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_seek_data_hole": "false", 2023-10-09T04:26:31.362 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_sloppy_crc": "false", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_splice": "false", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_split_multiple": "2", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_split_rand_factor": "20", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_update_to": "1000", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T04:26:31.363 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_xfs_extsize": "false", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "filestore_zfs_snap": "false", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "fio_dir": "/tmp/fio", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "fsid": "4ea1f4e4-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:26:31.364 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_allow_other": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_big_writes": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_debug": "false", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_default_permissions": "false", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_disable_pagecache": "false", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_max_write": "0", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_multithreaded": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_require_active_mds": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_set_user_groups": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_splice_move": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_splice_read": "true", 2023-10-09T04:26:31.365 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_splice_write": "true", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "gss_target_name": "ceph", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "heartbeat_file": "", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "heartbeat_inject_failure": "0", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "heartbeat_interval": "5", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "host": "smithi082", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T04:26:31.366 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "inject_early_sigterm": "false", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "jaeger_agent_port": "6799", 2023-10-09T04:26:31.367 INFO:teuthology.orchestra.run.smithi082.stdout: "jaeger_tracing_enable": "false", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_aio": "true", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_align_min_size": "65536", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_block_align": "true", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_block_size": "4096", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_dio": "true", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_discard": "false", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_force_aio": "false", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_ignore_corruption": "false", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_max_write_entries": "100", 2023-10-09T04:26:31.368 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_replay_from": "0", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_write_header_frequency": "0", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journal_zero_on_create": "false", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journaler_prefetch_periods": "10", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journaler_prezero_periods": "5", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "journaler_write_head_interval": "15", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "key": "", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "keyfile": "", 2023-10-09T04:26:31.369 INFO:teuthology.orchestra.run.smithi082.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_backend": "rocksdb", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_max_bytes": "67108864", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_max_ops": "512", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_onode_map_size": "1024", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T04:26:31.370 INFO:teuthology.orchestra.run.smithi082.stdout: "kstore_sync_transaction": "false", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_block_size": "65536", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_bloom_size": "0", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_cache_size": "536870912", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_compression": "false", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_log": "", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_max_open_files": "0", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_paranoid": "false", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "librados_thread_count": "2", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "lockdep": "false", 2023-10-09T04:26:31.371 INFO:teuthology.orchestra.run.smithi082.stdout: "lockdep_force_backtrace": "false", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_coarse_timestamps": "true", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_flush_on_exit": "false", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_graylog_port": "12201", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_max_new": "1000", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_max_recent": "10000", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_stderr_prefix": "debug ", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_to_file": "true", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_to_graylog": "false", 2023-10-09T04:26:31.372 INFO:teuthology.orchestra.run.smithi082.stdout: "log_to_journald": "false", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "log_to_stderr": "false", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "log_to_syslog": "false", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_action_on_write_error": "1", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_alternate_name_max": "8192", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_asio_thread_count": "2", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_export_pin": "true", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T04:26:31.373 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_interval": "10", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_max": "-1", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_max_until": "-1", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_merge_size": "50", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T04:26:31.374 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_mode": "0", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_split_bits": "3", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_split_size": "10000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T04:26:31.375 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_mid": "0.700000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_trim_interval": "1", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T04:26:31.376 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_debug_auth_pins": "false", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_debug_frag": "false", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_debug_scatterstat": "false", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_debug_subtrees": "false", 2023-10-09T04:26:31.377 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_default_dir_hash": "2", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_defer_session_stale": "true", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dir_max_entries": "0", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dir_prefetch": "true", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T04:26:31.378 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_early_reply": "true", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_enable_op_tracker": "true", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_enforce_unique_name": "true", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T04:26:31.379 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_health_dummy": "false", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T04:26:31.380 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_join_fs": "", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_journal_format": "1", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_create_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_export_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_import_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_journal_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_link_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_openc_at": "0", 2023-10-09T04:26:31.381 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_rename_at": "0", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_max_events": "-1", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_max_segments": "128", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_pause": "false", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_segment_size": "0", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T04:26:31.382 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_completed_requests": "100000", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_export_size": "20971520", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_file_recover": "32", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_purge_files": "64", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_purge_ops": "8192", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_metrics_update_interval": "2", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_min_caps_per_client": "100", 2023-10-09T04:26:31.383 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_numa_node": "-1", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_op_history_duration": "600", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_op_history_size": "20", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_op_log_threshold": "5", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_ping_grace": "15", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_ping_interval": "5", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T04:26:31.384 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_max_caps": "30000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_replay_interval": "1.000000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_root_ino_gid": "0", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_root_ino_uid": "0", 2023-10-09T04:26:31.385 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_shutdown_check": "0", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_skip_ino": "0", 2023-10-09T04:26:31.386 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_snap_min_uid": "0", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_snap_rstat": "false", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_symlink_recovery": "true", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_thrash_exports": "0", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_thrash_fragments": "0", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_tick_interval": "5.000000", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_valgrind_exit": "false", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_verify_backtrace": "1", 2023-10-09T04:26:31.387 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_verify_scatter": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mds_wipe_sessions": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mempool_debug": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "memstore_page_set": "false", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "memstore_page_size": "65536", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_client_bytes": "134217728", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_client_messages": "512", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T04:26:31.388 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_disabled_modules": "", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_mds_messages": "128", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_mon_messages": "128", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T04:26:31.389 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_osd_messages": "8192", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_pool": "true", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_standby_modules": "true", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_stats_period": "5", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_stats_threshold": "5", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_tick_period": "2", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_allow_pool_delete": "true", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T04:26:31.390 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_bytes": "104857600", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T04:26:31.391 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_journald": "false", 2023-10-09T04:26:31.392 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_compact_on_start": "false", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_compact_on_trim": "true", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_cpu_threads": "4", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T04:26:31.393 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_data_avail_crit": "5", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_data_avail_warn": "5", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_dump_json": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_extra_checks": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T04:26:31.394 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_election_default_strategy": "1", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_election_timeout": "5.000000", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T04:26:31.395 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_enable_op_tracker": "true", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_fake_pool_delete": "false", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_force_quorum_join": "false", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_log_update_period": "5", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_max_detail": "50", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_to_clog": "true", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_host": "[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] [v2:172.21.15.82:3301/0,v1:172.21.15.82:6790/0]", 2023-10-09T04:26:31.396 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_host_override": "", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_initial_members": "", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_lease": "5.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_log_full_interval": "50", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_log_max": "10000", 2023-10-09T04:26:31.397 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_log_max_summary": "50", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_log_epochs": "500", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_osd": "10000", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T04:26:31.398 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_memory_autotune": "true", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_memory_target": "2147483648", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_digest_period": "5", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_complaint_time": "30", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_history_duration": "600", 2023-10-09T04:26:31.399 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_history_size": "20", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_op_log_threshold": "5", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T04:26:31.400 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_cache_size": "500", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T04:26:31.401 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T04:26:31.402 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_report_timeout": "900", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T04:26:31.403 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_reweight_max_osds": "4", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T04:26:31.404 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_scrub_interval": "86400", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_scrub_max_keys": "100", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_scrub_timeout": "300", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_session_timeout": "300", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_smart_report_timeout": "5", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_stretch_pool_size": "4", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T04:26:31.405 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_debug": "false", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_tick_interval": "5", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T04:26:31.406 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_older_version": "true", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T04:26:31.407 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "monmap": "", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T04:26:31.408 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "motr_tracing_enabled": "false", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_op_threads": "3", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_cm": "false", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_device_name": "", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T04:26:31.409 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_sl": "3", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_rdma_type": "ib", 2023-10-09T04:26:31.410 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_async_reap_threshold": "5", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_before_connect": "false", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_ipv4": "true", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_ipv6": "false", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_msgr1": "true", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_msgr2": "true", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_port_max": "7568", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_port_min": "6800", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_retry_count": "3", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_bind_retry_delay": "5", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_blackhole_client": "false", 2023-10-09T04:26:31.411 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_blackhole_mds": "false", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_blackhole_mgr": "false", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_blackhole_mon": "false", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_blackhole_osd": "false", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_client_mode": "crc secure", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_cluster_type": "", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_compress_secure": "false", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_crc_data": "true", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_crc_header": "true", 2023-10-09T04:26:31.412 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_die_on_bug": "true", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_die_on_old_message": "true", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_hugepages": "", 2023-10-09T04:26:31.413 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_lro": "true", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_pmd": "", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_port_id": "0", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_dump_on_send": "false", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T04:26:31.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_delay_type": "", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_network_congestion": "0", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_inject_socket_failures": "0", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_max_accept_failures": "4", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_max_backoff": "15.000000", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T04:26:31.415 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_osd_compress_mode": "none", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_pq_min_cost": "65536", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_public_type": "", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_service_mode": "crc secure", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_tcp_nodelay": "true", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T04:26:31.416 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "ms_type": "async+posix", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "no_config_file": "false", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_inflight_ops": "1024", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objecter_timeout": "10.000000", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "objectstore_blackhole": "false", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "openssl_engine_opts": "", 2023-10-09T04:26:31.417 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_max_ops": "4", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_agent_slop": "0.020000", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_asio_thread_count": "2", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T04:26:31.418 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backfill_scan_max": "512", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backfill_scan_min": "64", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backoff_on_peering": "false", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_beacon_report_interval": "300", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_bench_duration": "30", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T04:26:31.419 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_blkin_trace_all": "false", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_class_update_on_start": "true", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_client_message_cap": "256", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T04:26:31.420 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_client_op_priority": "63", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_client_watch_timeout": "30", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_command_max_records": "256", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_compact_on_start": "false", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_crush_update_on_start": "true", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T04:26:31.421 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_op_order": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T04:26:31.422 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_shutdown": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T04:26:31.423 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_default_notify_timeout": "30", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T04:26:31.424 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_enable_op_tracker": "true", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_fast_info": "true", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_fast_shutdown": "true", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T04:26:31.425 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_function_tracing": "false", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_grace": "20", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_interval": "6", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_stale": "600", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T04:26:31.426 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_journal_size": "5120", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_kill_backfill_at": "0", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_map_cache_size": "50", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_map_dedup": "true", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_map_message_max": "40", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T04:26:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_attr_name_len": "100", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_attr_size": "0", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_backfills": "1", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_markdown_count": "5", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_markdown_period": "600", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_object_name_len": "2048", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_object_size": "134217728", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T04:26:31.428 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_pgls": "1024", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_push_cost": "8388608", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_push_objects": "10", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_scrubs": "1", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_max_write_size": "90", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T04:26:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_profile": "balanced", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T04:26:31.430 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_base": "805306368", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T04:26:31.431 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_target": "4294967296", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_target_autotune": "false", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_min_recovery_priority": "0", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mon_report_interval": "5", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_num_cache_shards": "32", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T04:26:31.432 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_numa_node": "-1", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_objecter_finishers": "1", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_objectstore": "bluestore", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_objectstore_fuse": "false", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_objectstore_tracing": "false", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_history_duration": "600", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_history_size": "20", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_log_threshold": "5", 2023-10-09T04:26:31.433 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_shards": "0", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_op_thread_timeout": "15", 2023-10-09T04:26:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_open_classes_on_start": "true", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_os_flags": "0", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_peering_op_priority": "255", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_delete_priority": "5", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T04:26:31.435 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_crimson": "false", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T04:26:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_flags": "0", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_min_size": "0", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_size": "3", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_default_type": "replicated", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T04:26:31.437 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_cost": "20971520", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_active": "0", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T04:26:31.438 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_op_priority": "3", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_priority": "5", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_repair_during_recovery": "false", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T04:26:31.439 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_cost": "52428800", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T04:26:31.440 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_end_hour": "0", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_priority": "5", 2023-10-09T04:26:31.441 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_skip_data_digest": "false", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_smart_report_timeout": "5", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_priority": "5", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T04:26:31.442 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_target_transaction_size": "30", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T04:26:31.443 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_tracing": "false", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_use_stale_snap": "false", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_kill_at": "0", 2023-10-09T04:26:31.444 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_max_join_drift": "10", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_min": "500", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_min_wait": "0.050000", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_service_trim_max": "500", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_service_trim_min": "250", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_stash_full_interval": "25", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_trim_max": "500", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "paxos_trim_min": "250", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "perf": "true", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "pid_file": "", 2023-10-09T04:26:31.445 INFO:teuthology.orchestra.run.smithi082.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "public_addr": "-", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "public_addrv": "", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "public_bind_addr": "-", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "public_network_interface": "", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "qat_compressor_enabled": "false", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "rados_mon_op_timeout": "0", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "rados_osd_op_timeout": "0", 2023-10-09T04:26:31.446 INFO:teuthology.orchestra.run.smithi082.stdout: "rados_tracing": "false", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_atime_update_interval": "60", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache": "true", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T04:26:31.447 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_policy": "writearound", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_size": "33554432", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_compression_hint": "none", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_clone_format": "auto", 2023-10-09T04:26:31.448 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_data_pool": "", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_features": "61", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_format": "2", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_map_options": "", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_order": "22", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_pool": "rbd", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_stripe_count": "0", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T04:26:31.449 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_io_scheduler": "simple", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T04:26:31.450 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_order": "24", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_pool": "", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_journal_splay_width": "4", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T04:26:31.451 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T04:26:31.452 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T04:26:31.453 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_op_threads": "1", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_plugins": "", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T04:26:31.454 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T04:26:31.455 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T04:26:31.456 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_tracing": "false", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_validate_names": "true", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rbd_validate_pool": "true", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "restapi_base_url": "", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "restapi_log_level": "", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_admin_entry": "admin", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_backend_store": "rados", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_barbican_url": "", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_beast_enable_async": "true", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T04:26:31.457 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_cache_enabled": "true", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_config_store": "rados", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_content_length_compat": "false", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T04:26:31.458 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_cross_domain_policy": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T04:26:31.459 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T04:26:31.460 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T04:26:31.461 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T04:26:31.462 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T04:26:31.463 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_log_window": "30", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T04:26:31.464 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T04:26:31.465 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dns_name": "", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dns_s3website_name": "", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_ops_log": "false", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_static_website": "false", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enable_usage_log": "false", 2023-10-09T04:26:31.466 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_expose_bucket": "false", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_extended_http_attrs": "", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_filter": "none", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_objs": "32", 2023-10-09T04:26:31.467 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_init_timeout": "300", 2023-10-09T04:26:31.468 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_password": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_project": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_token": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_admin_user": "", 2023-10-09T04:26:31.469 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_api_version": "2", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_url": "", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_max_objs": "32", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_max_worker": "3", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T04:26:31.471 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_log_http_headers": "", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T04:26:31.472 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_attr_size": "0", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_listing_results": "1000", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_notify_retries": "10", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T04:26:31.473 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T04:26:31.474 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T04:26:31.475 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_num_control_oids": "8", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_numa_node": "-1", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T04:26:31.476 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_op_tracing": "false", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_opa_token": "", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_opa_url": "", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ops_log_rados": "false", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T04:26:31.477 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_print_continue": "true", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T04:26:31.478 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_rados_tracing": "false", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_realm": "", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_realm_id": "", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_region": "", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_request_uri": "", 2023-10-09T04:26:31.479 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_resolve_cname": "false", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_run_sync_thread": "true", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T04:26:31.480 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_script_uri": "", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_service_provider_name": "", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_client_id": "", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_client_secret": "", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_entry": "sts", 2023-10-09T04:26:31.481 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_key": "sts", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_auth_url": "", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_custom_header": "", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_need_stats": "true", 2023-10-09T04:26:31.482 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_tenant_name": "", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_url": "", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_lease_period": "120", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T04:26:31.483 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_thread_pool_size": "512", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_comment": "", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_createby": "", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_encoding": "", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_flag": "false", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_origin": "", 2023-10-09T04:26:31.484 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_torrent_tracker": "", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_usage_max_shards": "32", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_use_opa_authz": "false", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T04:26:31.485 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_user_unique_email": "true", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_verify_ssl": "true", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zone": "", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zone_id": "", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zonegroup": "", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zonegroup_id": "", 2023-10-09T04:26:31.486 INFO:teuthology.orchestra.run.smithi082.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_block_size": "4096", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T04:26:31.487 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_partition_filters": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_perf": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T04:26:31.488 INFO:teuthology.orchestra.run.smithi082.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "run_dir": "/var/run/ceph", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_block_create": "true", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_device_size": "53687091200", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T04:26:31.489 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_main_device_type": "SSD", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "seastore_segment_size": "67108864", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "set_keepcaps": "false", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "setgroup": "ceph", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "setuser": "ceph", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "setuser_match_path": "", 2023-10-09T04:26:31.490 INFO:teuthology.orchestra.run.smithi082.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T04:26:31.491 INFO:teuthology.orchestra.run.smithi082.stdout: "thp": "false", 2023-10-09T04:26:31.491 INFO:teuthology.orchestra.run.smithi082.stdout: "threadpool_default_timeout": "60", 2023-10-09T04:26:31.491 INFO:teuthology.orchestra.run.smithi082.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T04:26:31.491 INFO:teuthology.orchestra.run.smithi082.stdout: "throttler_perf_counter": "true" 2023-10-09T04:26:31.491 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:26:32.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T04:26:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:32 smithi153 bash[59565]: cluster 2023-10-09T04:26:31.335856+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:32 smithi082 bash[77063]: cluster 2023-10-09T04:26:31.335856+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:32 smithi082 bash[79733]: cluster 2023-10-09T04:26:31.335856+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:34 smithi153 bash[59565]: cluster 2023-10-09T04:26:33.336421+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:34.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:34 smithi082 bash[77063]: cluster 2023-10-09T04:26:33.336421+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:34.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:34 smithi082 bash[79733]: cluster 2023-10-09T04:26:33.336421+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:36 smithi153 bash[59565]: cluster 2023-10-09T04:26:35.337482+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:36.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:36 smithi082 bash[77063]: cluster 2023-10-09T04:26:35.337482+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:36.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:36 smithi082 bash[79733]: cluster 2023-10-09T04:26:35.337482+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:36.893 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:38.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:38 smithi153 bash[59565]: cluster 2023-10-09T04:26:37.338323+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:38.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:38 smithi082 bash[77063]: cluster 2023-10-09T04:26:37.338323+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:38.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:38 smithi082 bash[79733]: cluster 2023-10-09T04:26:37.338323+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:39.695 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a: {} 2023-10-09T04:26:39.712 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b: {} 2023-10-09T04:26:39.729 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c: {} 2023-10-09T04:26:39.731 INFO:teuthology.orchestra.run.smithi082.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:26:39.731 INFO:teuthology.orchestra.run.smithi082.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:26:39.731 INFO:teuthology.orchestra.run.smithi082.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:26:39.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:39] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.33.4" 2023-10-09T04:26:40.494 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T04:26:40.494 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T04:26:40.495 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:26:40.497 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:26:40.497 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T04:26:40.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T04:26:40.498 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T04:26:40.498 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:26:40.498 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:26:40.498 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T04:26:40.498 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:40.499 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T04:26:40.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:26:40.499 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T04:26:40.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:26:40.500 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T04:26:40.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:26:40.533 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:40.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:40.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:40.650 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:40 smithi153 bash[59565]: cluster 2023-10-09T04:26:39.339031+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:40.650 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:40 smithi153 bash[59565]: audit 2023-10-09T04:26:39.675121+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.82:0/2614127129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:26:40.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:40.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:40 smithi082 bash[79733]: cluster 2023-10-09T04:26:39.339031+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:40 smithi082 bash[79733]: audit 2023-10-09T04:26:39.675121+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.82:0/2614127129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:26:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:40 smithi082 bash[77063]: cluster 2023-10-09T04:26:39.339031+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:40 smithi082 bash[77063]: audit 2023-10-09T04:26:39.675121+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.82:0/2614127129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:26:40.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:40.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:40.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:40.911 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:40.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:40.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:40.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:41.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:41.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:41.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:41.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:41.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:41.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:41.353 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:41.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:41.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:41.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:41.625 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:41.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:41.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:41.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:41.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:41.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:41.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:42.029 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:42.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:42.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:42.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:42.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:42.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:42.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:42 smithi153 bash[59565]: cluster 2023-10-09T04:26:41.340001+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:42.687 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:42 smithi082 bash[77063]: cluster 2023-10-09T04:26:41.340001+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:42.687 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:42 smithi082 bash[79733]: cluster 2023-10-09T04:26:41.340001+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:42.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:42.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:42.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:42.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:42.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:43.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:43.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:43.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:43.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:43.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:43.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:43.610 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:43.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:43.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:43.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:43.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:43.914 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:43.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:43.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:44.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:44.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:44.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:44.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:44.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:44.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:44.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:44.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:44.459 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:44.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:44.576 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:44 smithi082 bash[77063]: cluster 2023-10-09T04:26:43.340908+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:44.576 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:44 smithi082 bash[77063]: audit 2023-10-09T04:26:44.255545+0000 mon.a (mon.0) 530 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.576 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:44 smithi082 bash[79733]: cluster 2023-10-09T04:26:43.340908+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:44.577 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:44 smithi082 bash[79733]: audit 2023-10-09T04:26:44.255545+0000 mon.a (mon.0) 530 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:44 smithi153 bash[59565]: cluster 2023-10-09T04:26:43.340908+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:44 smithi153 bash[59565]: audit 2023-10-09T04:26:44.255545+0000 mon.a (mon.0) 530 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:44.735 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:44.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:44.868 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:44.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:44.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:45.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:45.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:45.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:45.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:45.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:45.287 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:45.288 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:45.289 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:45.289 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:45.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:45.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:45.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:45.679 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:45.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:45.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:45.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:45.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:45.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:46.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:46.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:46.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:46.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:46.321 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:46.458 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.470 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:46 smithi153 bash[59565]: cluster 2023-10-09T04:26:45.341968+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:46.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:46.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:46.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:46.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:46 smithi082 bash[77063]: cluster 2023-10-09T04:26:45.341968+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:46 smithi082 bash[79733]: cluster 2023-10-09T04:26:45.341968+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:46.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:46.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:46.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:46.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:46.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:47.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:47.117 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:47.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:47.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:47.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:47.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:47.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:47.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:47.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:47.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:47.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:47.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:47.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:47.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:47.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:48.048 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:48.182 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:48.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:48.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:48.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:48.454 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:48.466 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:48 smithi153 bash[59565]: cluster 2023-10-09T04:26:47.342682+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:48.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:48.544 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:48 smithi082 bash[77063]: cluster 2023-10-09T04:26:47.342682+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:48.544 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:48 smithi082 bash[79733]: cluster 2023-10-09T04:26:47.342682+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:48.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:48.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:48.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:48.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:48.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:48.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:48.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:49.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:49.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:49.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:49.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:49.379 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:49.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:49.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:49.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:49.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:49.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:49.785 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:49.926 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:49.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:49.952 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:49] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.33.4" 2023-10-09T04:26:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:49.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:50.054 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:50.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:50.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:50.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:50.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:50.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:50.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:50.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:50.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:50.504 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:50 smithi082 bash[77063]: cluster 2023-10-09T04:26:49.343356+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:50.504 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:50 smithi082 bash[79733]: cluster 2023-10-09T04:26:49.343356+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:50.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:50.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:50 smithi153 bash[59565]: cluster 2023-10-09T04:26:49.343356+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:50.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:50.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:50.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:50.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:50.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:50.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:50.957 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:50.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:50.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:51.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:51.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:51.219 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:51.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:51.348 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:51.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:51.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:51.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:51.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:51.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:51.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:51.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:51.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:52.026 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:52.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:52.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:52.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.326 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:52.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:52 smithi153 bash[59565]: cluster 2023-10-09T04:26:51.344023+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:52.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.460 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:52.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:52.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:52.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:52.650 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:52 smithi082 bash[77063]: cluster 2023-10-09T04:26:51.344023+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:52.650 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:52 smithi082 bash[79733]: cluster 2023-10-09T04:26:51.344023+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:52.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:52.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:52.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:52.817 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:52.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:52.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:52.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:52.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:52.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:53.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:53.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:53.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:53.255 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:53.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:53.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:53.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:53.419 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:26:53.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:53.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:53 smithi082 bash[79733]: audit 2023-10-09T04:26:53.010362+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.82:0/2152531401' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.502 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:53 smithi082 bash[79733]: audit 2023-10-09T04:26:53.010838+0000 mon.a (mon.0) 531 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.502 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:53 smithi082 bash[77063]: audit 2023-10-09T04:26:53.010362+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.82:0/2152531401' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.502 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:53 smithi082 bash[77063]: audit 2023-10-09T04:26:53.010838+0000 mon.a (mon.0) 531 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:53.565 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:26:53.587 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:26:53.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:53.626 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:26:53.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:53.648 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:26:53.663 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:53.663 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:53 smithi153 bash[59565]: audit 2023-10-09T04:26:53.010362+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.82:0/2152531401' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:53 smithi153 bash[59565]: audit 2023-10-09T04:26:53.010838+0000 mon.a (mon.0) 531 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:53.684 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:26:53.713 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:26:53.736 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:26:53.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:53.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:53.765 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:26:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.768 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:26:53.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:53.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:53.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:53.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:54.019 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:54.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:54.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:54.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:54.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:54.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:54.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:54.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:54.425 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_0' created 2023-10-09T04:26:54.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:54.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:54.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:54.672 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: cluster 2023-10-09T04:26:53.344700+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: cluster 2023-10-09T04:26:53.399039+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: cluster 2023-10-09T04:26:53.408364+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: cluster 2023-10-09T04:26:53.410630+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: audit 2023-10-09T04:26:53.410721+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: cluster 2023-10-09T04:26:53.410817+0000 mon.a (mon.0) 534 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: audit 2023-10-09T04:26:53.526087+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.82:0/3969736768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: audit 2023-10-09T04:26:53.935707+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.82:0/3747332673' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:54 smithi153 bash[59565]: audit 2023-10-09T04:26:53.936322+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:54.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: cluster 2023-10-09T04:26:53.344700+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: cluster 2023-10-09T04:26:53.399039+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: cluster 2023-10-09T04:26:53.408364+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: cluster 2023-10-09T04:26:53.410630+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: audit 2023-10-09T04:26:53.410721+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: cluster 2023-10-09T04:26:53.410817+0000 mon.a (mon.0) 534 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: audit 2023-10-09T04:26:53.526087+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.82:0/3969736768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: audit 2023-10-09T04:26:53.935707+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.82:0/3747332673' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[77063]: audit 2023-10-09T04:26:53.936322+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: cluster 2023-10-09T04:26:53.344700+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: cluster 2023-10-09T04:26:53.399039+0000 mon.a (mon.0) 532 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: cluster 2023-10-09T04:26:53.408364+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: cluster 2023-10-09T04:26:53.410630+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: audit 2023-10-09T04:26:53.410721+0000 mon.a (mon.0) 533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: cluster 2023-10-09T04:26:53.410817+0000 mon.a (mon.0) 534 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: audit 2023-10-09T04:26:53.526087+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.82:0/3969736768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: audit 2023-10-09T04:26:53.935707+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.82:0/3747332673' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:54 smithi082 bash[79733]: audit 2023-10-09T04:26:53.936322+0000 mon.a (mon.0) 536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:26:54.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:54.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:54.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:54.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:54.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:55.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.073 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:55.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:55.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:55.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:55.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:55.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:55.450 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[77063]: cluster 2023-10-09T04:26:54.416602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.450 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[77063]: cluster 2023-10-09T04:26:54.416735+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[77063]: cluster 2023-10-09T04:26:54.416890+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[77063]: audit 2023-10-09T04:26:54.418283+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[77063]: cluster 2023-10-09T04:26:54.418377+0000 mon.a (mon.0) 538 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[79733]: cluster 2023-10-09T04:26:54.416602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[79733]: cluster 2023-10-09T04:26:54.416735+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[79733]: cluster 2023-10-09T04:26:54.416890+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[79733]: audit 2023-10-09T04:26:54.418283+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:26:55.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:55 smithi082 bash[79733]: cluster 2023-10-09T04:26:54.418377+0000 mon.a (mon.0) 538 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:26:55.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:55.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:55.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:55 smithi153 bash[59565]: cluster 2023-10-09T04:26:54.416602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:55 smithi153 bash[59565]: cluster 2023-10-09T04:26:54.416735+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:55 smithi153 bash[59565]: cluster 2023-10-09T04:26:54.416890+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T04:26:55.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:55 smithi153 bash[59565]: audit 2023-10-09T04:26:54.418283+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:26:55.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:55 smithi153 bash[59565]: cluster 2023-10-09T04:26:54.418377+0000 mon.a (mon.0) 538 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:26:55.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:55.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:55.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:55.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:55.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:55.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:55.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:56.001 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:56.105 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:56.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:56.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:56.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:56.498 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.345368+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.421468+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.424805+0000 mon.a (mon.0) 539 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.424853+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.425573+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[79733]: cluster 2023-10-09T04:26:55.425651+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.345368+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.421468+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.571 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.424805+0000 mon.a (mon.0) 539 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:26:56.572 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.424853+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.572 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.425573+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.572 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:56 smithi082 bash[77063]: cluster 2023-10-09T04:26:55.425651+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:26:56.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:56.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T04:26:56.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:56.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.345368+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.421468+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.424805+0000 mon.a (mon.0) 539 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:26:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.424853+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.425573+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:56 smithi153 bash[59565]: cluster 2023-10-09T04:26:55.425651+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T04:26:56.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:56.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:56.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:56.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:57.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:57.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:57.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:26:57.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:57.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:57.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:57.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:57.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:57.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:57.591 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[77063]: cluster 2023-10-09T04:26:56.423870+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[77063]: cluster 2023-10-09T04:26:56.426011+0000 mon.a (mon.0) 540 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[77063]: cluster 2023-10-09T04:26:56.496525+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[77063]: cluster 2023-10-09T04:26:56.520584+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[77063]: cluster 2023-10-09T04:26:57.148629+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[79733]: cluster 2023-10-09T04:26:56.423870+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[79733]: cluster 2023-10-09T04:26:56.426011+0000 mon.a (mon.0) 540 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[79733]: cluster 2023-10-09T04:26:56.496525+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[79733]: cluster 2023-10-09T04:26:56.520584+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.592 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:57 smithi082 bash[79733]: cluster 2023-10-09T04:26:57.148629+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:57.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:57.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:57.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:57.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:57 smithi153 bash[59565]: cluster 2023-10-09T04:26:56.423870+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:57 smithi153 bash[59565]: cluster 2023-10-09T04:26:56.426011+0000 mon.a (mon.0) 540 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:26:57.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:57 smithi153 bash[59565]: cluster 2023-10-09T04:26:56.496525+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:57 smithi153 bash[59565]: cluster 2023-10-09T04:26:56.520584+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:57 smithi153 bash[59565]: cluster 2023-10-09T04:26:57.148629+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T04:26:57.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:57.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:57.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:58.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:58.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.134 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:58.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:58.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:58.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:58.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:58.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:58.550 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:58.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:26:58.597 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:26:58 smithi082 bash[77063]: cluster 2023-10-09T04:26:57.346102+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:58.598 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:26:58 smithi082 bash[79733]: cluster 2023-10-09T04:26:57.346102+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:58.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:26:58 smithi153 bash[59565]: cluster 2023-10-09T04:26:57.346102+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:26:58.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:58.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:58.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:58.917 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.947 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:58.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:59.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:59.081 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:59.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:59.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:59.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:59.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:59.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:59.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:59.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:59.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:59.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:59.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:59.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:59.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:26:59.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:26:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:26:59] "GET /metrics HTTP/1.1" 200 36240 "" "Prometheus/2.33.4" 2023-10-09T04:26:59.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:00.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:00.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:00.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:00.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:00.268 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[77063]: audit 2023-10-09T04:26:59.261811+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:00.268 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[77063]: audit 2023-10-09T04:26:59.263563+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:00.268 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[77063]: cluster 2023-10-09T04:26:59.346652+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v312: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:00.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:00.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:00 smithi153 bash[59565]: audit 2023-10-09T04:26:59.261811+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:00 smithi153 bash[59565]: audit 2023-10-09T04:26:59.263563+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:00 smithi153 bash[59565]: cluster 2023-10-09T04:26:59.346652+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v312: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:00.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:00.567 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:00.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:00.682 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[79733]: audit 2023-10-09T04:26:59.261811+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:00.682 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[79733]: audit 2023-10-09T04:26:59.263563+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:00.682 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:00 smithi082 bash[79733]: cluster 2023-10-09T04:26:59.346652+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v312: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:00.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:00.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:00.840 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:00.943 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:00.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:00.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:01.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:01.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:01.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:01.217 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:01.247 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:01.343 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:01.345 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:01.346 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:01.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.375 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:01.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:01.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:01.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:01.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:01.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:01.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:01.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:01.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:02.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:02.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:02.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:02.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:02.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:02.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:02.357 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:02.405 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:02.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:02.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:02.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:02.563 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:02 smithi082 bash[77063]: cluster 2023-10-09T04:27:01.347414+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:02.563 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:02 smithi082 bash[79733]: cluster 2023-10-09T04:27:01.347414+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:02.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:02.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:02 smithi153 bash[59565]: cluster 2023-10-09T04:27:01.347414+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:02.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:02.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:02.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:02.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:02.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:03.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:03.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:03.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:03.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:03.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:03.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:03.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:03.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:03.581 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:03.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:03.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:03.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:03.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:03.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:04.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:04.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:04.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:04.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:04.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:04.488 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:04.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:04.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.632 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:04 smithi082 bash[77063]: cluster 2023-10-09T04:27:03.347954+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:04.632 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:04 smithi082 bash[79733]: cluster 2023-10-09T04:27:03.347954+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:04.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:04.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:04 smithi153 bash[59565]: cluster 2023-10-09T04:27:03.347954+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:04.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:04.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:04.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:04.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:04.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:05.014 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:05.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:05.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:05.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:05.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:05.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:05.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:05.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:05.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:05.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:05.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:05.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:05.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:05.845 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:05.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:05.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:05.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:06.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:06.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:06.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:06.247 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:06.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:06.379 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:06 smithi082 bash[77063]: cluster 2023-10-09T04:27:05.349037+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:06 smithi082 bash[79733]: cluster 2023-10-09T04:27:05.349037+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:06.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:06.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:06.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:06.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:06.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:06.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:06 smithi153 bash[59565]: cluster 2023-10-09T04:27:05.349037+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:06.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:06.776 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:06.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:06.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:06.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:07.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:07.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:07.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:07.292 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:07.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:07.520 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:07.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:07.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:07.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:07.939 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:07.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:08.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:08.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:08.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:08.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.324 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:08.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:08.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:08.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:08.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:08.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:08.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:08 smithi153 bash[59565]: cluster 2023-10-09T04:27:07.349581+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:08.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:08.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:08 smithi082 bash[77063]: cluster 2023-10-09T04:27:07.349581+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:08.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:08 smithi082 bash[79733]: cluster 2023-10-09T04:27:07.349581+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:08.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:08.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:08.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:08.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:08.989 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:08.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:09.093 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:09.115 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:09.218 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:09.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:09.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:09.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:09.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:09.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:09.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:09.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:09.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:09.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:09.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:09.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:09.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:09] "GET /metrics HTTP/1.1" 200 36240 "" "Prometheus/2.33.4" 2023-10-09T04:27:10.010 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:10.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:10.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:10.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:10.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:10.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:10.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:10.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:10.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:10.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:10.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:10 smithi082 bash[77063]: cluster 2023-10-09T04:27:09.350098+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:10.535 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:10 smithi082 bash[77063]: audit 2023-10-09T04:27:09.424388+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:10.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:10 smithi082 bash[79733]: cluster 2023-10-09T04:27:09.350098+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:10.535 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:10 smithi082 bash[79733]: audit 2023-10-09T04:27:09.424388+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:10.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:10.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:10.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:10 smithi153 bash[59565]: cluster 2023-10-09T04:27:09.350098+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:10.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:10 smithi153 bash[59565]: audit 2023-10-09T04:27:09.424388+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:10.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:10.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:10.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:10.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:10.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:10.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:11.026 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:11.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:11.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:11.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:11.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:11.251 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:11.252 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:11.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:11.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:11.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:11.444 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:27:11.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:11.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:11.563 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:11 smithi082 bash[77063]: audit 2023-10-09T04:27:10.698957+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.82:0/286013431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.563 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:11 smithi082 bash[77063]: audit 2023-10-09T04:27:10.699319+0000 mon.a (mon.0) 544 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.563 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:11 smithi082 bash[79733]: audit 2023-10-09T04:27:10.698957+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.82:0/286013431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.563 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:11 smithi082 bash[79733]: audit 2023-10-09T04:27:10.699319+0000 mon.a (mon.0) 544 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.579 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:11.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:11.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:11.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:11.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:11.834 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:11 smithi153 bash[59565]: audit 2023-10-09T04:27:10.698957+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.82:0/286013431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.834 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:11 smithi153 bash[59565]: audit 2023-10-09T04:27:10.699319+0000 mon.a (mon.0) 544 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:11.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:11.952 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:27:11.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:11.979 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:27:11.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:11.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:11.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:11.998 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:27:12.017 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:27:12.039 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:27:12.063 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:27:12.086 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:27:12.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:12.111 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.114 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:27:12.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:12.221 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:12.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:12.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:12.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.350995+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.423078+0000 mon.a (mon.0) 545 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.433904+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: audit 2023-10-09T04:27:11.437040+0000 mon.a (mon.0) 546 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.437150+0000 mon.a (mon.0) 547 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.487217+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.925308+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: audit 2023-10-09T04:27:11.930193+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.82:0/3260368974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:12.487 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:12 smithi153 bash[59565]: cluster 2023-10-09T04:27:11.955122+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.508 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:12.612 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.350995+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.423078+0000 mon.a (mon.0) 545 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.433904+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: audit 2023-10-09T04:27:11.437040+0000 mon.a (mon.0) 546 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.437150+0000 mon.a (mon.0) 547 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.487217+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.925308+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: audit 2023-10-09T04:27:11.930193+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.82:0/3260368974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[77063]: cluster 2023-10-09T04:27:11.955122+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.350995+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v318: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.423078+0000 mon.a (mon.0) 545 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.433904+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: audit 2023-10-09T04:27:11.437040+0000 mon.a (mon.0) 546 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.437150+0000 mon.a (mon.0) 547 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.487217+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.925308+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: audit 2023-10-09T04:27:11.930193+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.82:0/3260368974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:12.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:12 smithi082 bash[79733]: cluster 2023-10-09T04:27:11.955122+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T04:27:12.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:12.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:12.893 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:12.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:12.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:13.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:13.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:13.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:13.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:13.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.412 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:13.446 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T04:27:13.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.527 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:13 smithi082 bash[79733]: audit 2023-10-09T04:27:12.441365+0000 mon.a (mon.0) 548 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:13.527 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:13 smithi082 bash[77063]: audit 2023-10-09T04:27:12.441365+0000 mon.a (mon.0) 548 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:13.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:13.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:13.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:13.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:13.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:13.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:13.910 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:13 smithi153 bash[59565]: audit 2023-10-09T04:27:12.441365+0000 mon.a (mon.0) 548 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:13.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:13.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:13.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:14.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:14.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:27:14.063 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:14.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:14.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:14.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:14.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:14.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:14.457 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:14.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.351677+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v320: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: audit 2023-10-09T04:27:13.439484+0000 mon.a (mon.0) 549 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.439545+0000 mon.a (mon.0) 550 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.529522+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.529728+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.529743+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[77063]: audit 2023-10-09T04:27:14.255251+0000 mon.a (mon.0) 551 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.351677+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v320: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: audit 2023-10-09T04:27:13.439484+0000 mon.a (mon.0) 549 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.439545+0000 mon.a (mon.0) 550 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.529522+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.529728+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.529743+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:14 smithi082 bash[79733]: audit 2023-10-09T04:27:14.255251+0000 mon.a (mon.0) 551 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:14.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:14.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.351677+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v320: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:14.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: audit 2023-10-09T04:27:13.439484+0000 mon.a (mon.0) 549 : audit 1 from='client.? 172.21.15.82:0/4117321169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:14.574 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.439545+0000 mon.a (mon.0) 550 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:27:14.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.529522+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.529728+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.529743+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:14.575 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:14 smithi153 bash[59565]: audit 2023-10-09T04:27:14.255251+0000 mon.a (mon.0) 551 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:14.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:14.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:14.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:14.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:14.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:14.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:14.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:14.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:14.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:15.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:15.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:15.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:15.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:15.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:15.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:15.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:15.515 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:15 smithi082 bash[77063]: cluster 2023-10-09T04:27:13.529403+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:15.516 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:15 smithi082 bash[79733]: cluster 2023-10-09T04:27:13.529403+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:15.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:15.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:15.643 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:15.657 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:15 smithi153 bash[59565]: cluster 2023-10-09T04:27:13.529403+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T04:27:15.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:15.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:15.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:15.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:15.899 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T04:27:15.900 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T04:27:15.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:27:15.902 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:27:15.902 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:27:15.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:15.943 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:27:15.943 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:27:15.943 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_106363 2023-10-09T04:27:15.943 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:27:15.943 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:27:15.957 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:15.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:16.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:16.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:27:16.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:16.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:16.229 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:16.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:16.334 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:16.360 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:16.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:16.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:27:16.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:16.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:16.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:16.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:16 smithi082 bash[77063]: cluster 2023-10-09T04:27:15.352743+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:16.615 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:16 smithi082 bash[79733]: cluster 2023-10-09T04:27:15.352743+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:16.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:16.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:16.741 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:16.749 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:16 smithi153 bash[59565]: cluster 2023-10-09T04:27:15.352743+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v322: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:27:16.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:16.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:16.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:16.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:16.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 16 2678 2662 166.365 166.375 0.00360364 0.00597385 2023-10-09T04:27:17.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:17.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:17.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:17.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:17.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:17.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:17.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:17.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:17.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:17.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:17.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:17.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:17.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:17.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:17.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:17.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:17.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 16 5693 5677 177.388 188.438 0.005326 0.00562062 2023-10-09T04:27:17.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:17.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:18.057 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:18.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:18.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:18.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:18.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:18.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:18.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:18.478 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:18.581 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:18.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:18 smithi153 bash[59565]: cluster 2023-10-09T04:27:17.353562+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 65 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T04:27:18.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:18.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:18 smithi082 bash[77063]: cluster 2023-10-09T04:27:17.353562+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 65 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T04:27:18.722 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:18 smithi082 bash[79733]: cluster 2023-10-09T04:27:17.353562+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 65 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s wr, 120 op/s 2023-10-09T04:27:18.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:18.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:18.848 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:18.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:18.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:18.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 16 8510 8494 176.941 176.062 0.0102017 0.0056399 2023-10-09T04:27:18.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:18.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:19.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:19.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:19.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:19.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:19.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:19.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:19.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:19.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.900 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:19] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.33.4" 2023-10-09T04:27:19.901 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:19.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:19.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:19.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:19.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 15 11062 11047 172.578 159.562 0.0109576 0.00578424 2023-10-09T04:27:20.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:20.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:20.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:20.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:20.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:20.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:20.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:20.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:20 smithi082 bash[77063]: cluster 2023-10-09T04:27:19.354170+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 126 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s wr, 242 op/s 2023-10-09T04:27:20.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:20 smithi082 bash[79733]: cluster 2023-10-09T04:27:19.354170+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 126 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s wr, 242 op/s 2023-10-09T04:27:20.736 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:20.763 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:20.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:20.877 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:20 smithi153 bash[59565]: cluster 2023-10-09T04:27:19.354170+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 126 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s wr, 242 op/s 2023-10-09T04:27:20.889 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:20.890 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:20.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:20.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:20.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 13547 13531 169.11 155.25 0.0029706 0.00590032 2023-10-09T04:27:20.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:21.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:21.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:21.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:21.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:21.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:21.385 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:21.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:21.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:21.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:21.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:21.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:21.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:21.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:21.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:21.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:21.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:21.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:21.944 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 16 15948 15932 165.933 150.062 0.0102224 0.00601939 2023-10-09T04:27:22.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:22.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:22.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:22.212 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:22.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:22.346 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:22.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:22.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:22.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:22.614 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:22.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:22.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:22 smithi082 bash[77063]: cluster 2023-10-09T04:27:21.354757+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 445 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-10-09T04:27:22.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:22 smithi082 bash[77063]: audit 2023-10-09T04:27:22.237934+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:22.727 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:22 smithi082 bash[79733]: cluster 2023-10-09T04:27:21.354757+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 445 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-10-09T04:27:22.727 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:22 smithi082 bash[79733]: audit 2023-10-09T04:27:22.237934+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:22.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:22.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:22.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:22.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:22 smithi153 bash[59565]: cluster 2023-10-09T04:27:21.354757+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v325: 89 pgs: 89 active+clean; 445 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2023-10-09T04:27:22.854 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:22 smithi153 bash[59565]: audit 2023-10-09T04:27:22.237934+0000 mon.a (mon.0) 552 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:22.870 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:22.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:22.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 18078 18062 161.245 133.125 0.00622272 0.00619401 2023-10-09T04:27:22.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:23.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:23.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:23.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:23.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:23.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:23.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:23.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:23.499 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:23.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:23.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:23.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:23.658 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:23.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:23.788 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:23.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:23.923 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:23.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:23.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:23.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 20185 20169 157.548 131.688 0.00326513 0.00627883 2023-10-09T04:27:24.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:24.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:24.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:24.184 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:24.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:24.316 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:24.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:24.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:24.477 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:24 smithi082 bash[77063]: cluster 2023-10-09T04:27:23.355283+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 813 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:27:24.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:24.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:24.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:24 smithi153 bash[59565]: cluster 2023-10-09T04:27:23.355283+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 813 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:27:24.685 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:24.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:24.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:24 smithi082 bash[79733]: cluster 2023-10-09T04:27:23.355283+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 813 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:27:24.815 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:24.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:24.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 22279 22263 154.583 130.875 0.0133283 0.00646176 2023-10-09T04:27:24.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:24.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:24.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:24.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:25.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:25.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:25.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:25.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:25.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:25.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:25.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:25.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:25.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:25.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:25.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:25.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:25.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:25.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:25.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:25.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:25.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:25.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 24020 24004 150.005 108.812 0.00563731 0.0066586 2023-10-09T04:27:25.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:26.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:26.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:26.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:26.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:26.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:26.412 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:26.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:26.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.654 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:26 smithi082 bash[77063]: cluster 2023-10-09T04:27:25.356130+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:26.654 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:26 smithi082 bash[79733]: cluster 2023-10-09T04:27:25.356130+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:26.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:26.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:26.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:26.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:26.917 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:26 smithi153 bash[59565]: cluster 2023-10-09T04:27:25.356130+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:26.935 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:26.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:26.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 16 25912 25896 147.118 118.25 0.0194464 0.00678981 2023-10-09T04:27:27.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:27.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:27.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:27.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:27.433 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:27.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:27.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:27.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:27.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:27.826 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:27.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:27.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 27575 27559 143.518 103.938 0.0129978 0.0069606 2023-10-09T04:27:27.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:27.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:28.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:28.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:28.252 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:28.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:28.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:28.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:28.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:28.635 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:28 smithi082 bash[77063]: cluster 2023-10-09T04:27:27.356576+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:28.635 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:28 smithi082 bash[79733]: cluster 2023-10-09T04:27:27.356576+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:28.653 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:28.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:28.765 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:28 smithi153 bash[59565]: cluster 2023-10-09T04:27:27.356576+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:28.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:28.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:28.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:28.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 29238 29222 140.473 103.938 0.0118128 0.0071118 2023-10-09T04:27:29.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:29.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:29.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:29.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:29.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:29.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:29.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:29.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.583 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:29.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:29.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:29.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:29.826 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[77063]: audit 2023-10-09T04:27:28.790597+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[77063]: audit 2023-10-09T04:27:28.799818+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[77063]: audit 2023-10-09T04:27:29.256181+0000 mon.a (mon.0) 555 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[77063]: cluster 2023-10-09T04:27:29.357122+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:29] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[79733]: audit 2023-10-09T04:27:28.790597+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:29.827 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[79733]: audit 2023-10-09T04:27:28.799818+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:29.828 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[79733]: audit 2023-10-09T04:27:29.256181+0000 mon.a (mon.0) 555 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:29.828 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:29 smithi082 bash[79733]: cluster 2023-10-09T04:27:29.357122+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:27:29.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:29.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:29.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 30914 30898 137.921 104.75 0.00695549 0.00724376 2023-10-09T04:27:29.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:29.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:29.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:30.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:29 smithi153 bash[59565]: audit 2023-10-09T04:27:28.790597+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:29 smithi153 bash[59565]: audit 2023-10-09T04:27:28.799818+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:29 smithi153 bash[59565]: audit 2023-10-09T04:27:29.256181+0000 mon.a (mon.0) 555 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:29 smithi153 bash[59565]: cluster 2023-10-09T04:27:29.357122+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:27:30.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:30.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:30.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:30.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:30.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:30.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:30.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:30.511 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:30.614 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:30.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:30.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:30.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:30.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:30.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:30.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:30.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 16 32631 32615 135.88 107.312 0.00219211 0.00734622 2023-10-09T04:27:31.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:31.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:31.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:31.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:31.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:31.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:31.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:31.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:31.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:31.573 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:31.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:31.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:31.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:31.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:31.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:31.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:31.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 34263 34247 133.762 102 0.00665025 0.00747002 2023-10-09T04:27:31.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:31.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:31.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:32.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:32.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:32.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:32.360 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:32.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:32.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:32.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:32.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:32 smithi153 bash[59565]: cluster 2023-10-09T04:27:31.357756+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1.7 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:27:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:32 smithi082 bash[77063]: cluster 2023-10-09T04:27:31.357756+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1.7 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:27:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:32 smithi082 bash[79733]: cluster 2023-10-09T04:27:31.357756+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1.7 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:27:32.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:32.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:32.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:32.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:32.945 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 16 36050 36034 132.463 111.688 0.0057919 0.00754268 2023-10-09T04:27:33.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:33.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:33.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:33.192 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:33.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:33.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:33.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:33.459 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:33.562 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:33.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:33.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:33.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:33.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:33.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:33.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:33.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 37407 37391 129.807 84.8125 0.00881739 0.0076964 2023-10-09T04:27:33.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:33.977 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:33.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:34.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:34.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:34.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:34.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:34.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:34.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[77063]: cluster 2023-10-09T04:27:33.358275+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T04:27:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[77063]: audit 2023-10-09T04:27:34.370187+0000 mon.a (mon.0) 556 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[77063]: audit 2023-10-09T04:27:34.379388+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[79733]: cluster 2023-10-09T04:27:33.358275+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T04:27:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[79733]: audit 2023-10-09T04:27:34.370187+0000 mon.a (mon.0) 556 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:34 smithi082 bash[79733]: audit 2023-10-09T04:27:34.379388+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:34.529 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:34.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:34.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:34.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:34 smithi153 bash[59565]: cluster 2023-10-09T04:27:33.358275+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T04:27:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:34 smithi153 bash[59565]: audit 2023-10-09T04:27:34.370187+0000 mon.a (mon.0) 556 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:34 smithi153 bash[59565]: audit 2023-10-09T04:27:34.379388+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:34.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:34.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:34.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:34.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:34.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:34.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 16 38635 38619 127.014 76.75 0.00331512 0.00781207 2023-10-09T04:27:35.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:35.072 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:35.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:35.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:35.336 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:35.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:35.452 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[77063]: audit 2023-10-09T04:27:34.913003+0000 mon.a (mon.0) 558 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:35.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[77063]: audit 2023-10-09T04:27:34.913838+0000 mon.a (mon.0) 559 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:35.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[77063]: audit 2023-10-09T04:27:34.918890+0000 mon.a (mon.0) 560 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:35.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[79733]: audit 2023-10-09T04:27:34.913003+0000 mon.a (mon.0) 558 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:35.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[79733]: audit 2023-10-09T04:27:34.913838+0000 mon.a (mon.0) 559 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:35.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:35 smithi082 bash[79733]: audit 2023-10-09T04:27:34.918890+0000 mon.a (mon.0) 560 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:35.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:35.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:35.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:35 smithi153 bash[59565]: audit 2023-10-09T04:27:34.913003+0000 mon.a (mon.0) 558 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:35 smithi153 bash[59565]: audit 2023-10-09T04:27:34.913838+0000 mon.a (mon.0) 559 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:35.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:35 smithi153 bash[59565]: audit 2023-10-09T04:27:34.918890+0000 mon.a (mon.0) 560 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:27:35.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:35.796 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:27:35.796 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:20164 5.2 e110/106 5:4fd33a14:::benchmark_data_smithi082_106363_object20163:head v 110'1316, mlcod=110'1314)", 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.860761+0000", 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 11.931709243, 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12152721, 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.797 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860761+0000", 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860761+0000", 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.798 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860761+0000", 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860791+0000", 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0026e-05 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860798+0000", 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1119999999999998e-06 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.799 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860802+0000", 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.189e-06 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860836+0000", 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3912000000000003e-05 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.800 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.860849+0000", 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3122000000000001e-05 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982245+0000", 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.121395541 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982288+0000", 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3872000000000003e-05 2023-10-09T04:27:35.801 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:20173 5.3 5:cda2a92b:::benchmark_data_smithi082_106363_object20172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.861075+0000", 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 11.931395623, 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.122973891, 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.802 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 20173 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861075+0000", 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.803 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861075+0000", 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861074+0000", 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861096+0000", 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2419e-05 2023-10-09T04:27:35.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861104+0000", 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0050000000000004e-06 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861110+0000", 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9139999999999996e-06 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861164+0000", 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4481000000000002e-05 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861246+0000", 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1384000000000005e-05 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861328+0000", 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1772000000000002e-05 2023-10-09T04:27:35.806 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982409+0000", 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.121081885 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982452+0000", 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2885e-05 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.807 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.983956+0000", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001503814 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.984011+0000", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5164999999999998e-05 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.984049+0000", 2023-10-09T04:27:35.808 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7478999999999998e-05 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:20168 5.e e110/106 5:73f6c3c2:::benchmark_data_smithi082_106363_object20167:head v 110'1269, mlcod=110'1267)", 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.861655+0000", 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 11.930815214999999, 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.121779895, 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.809 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861655+0000", 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861655+0000", 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.810 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861655+0000", 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861740+0000", 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5050999999999996e-05 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.979747+0000", 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.118007522 2023-10-09T04:27:35.811 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.979774+0000", 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.637e-05 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981470+0000", 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001696632 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.812 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981493+0000", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2643e-05 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.983390+0000", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0018968640000000001 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.983435+0000", 2023-10-09T04:27:35.813 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5420000000000002e-05 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:20171 5.2 e110/106 5:4d94bd46:::benchmark_data_smithi082_106363_object20170:head v 110'1317, mlcod=110'1314)", 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.861716+0000", 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 11.930754215, 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.121661863, 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.814 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861716+0000", 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861716+0000", 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.815 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861716+0000", 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.861807+0000", 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1532000000000004e-05 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.979759+0000", 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.117952134 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.979785+0000", 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5936e-05 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981295+0000", 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0015095760000000001 2023-10-09T04:27:35.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981315+0000", 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.048e-05 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.983262+0000", 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019469719999999999 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.983378+0000", 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000115806 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:32666 5.3 5:caa4e5f7:::benchmark_data_smithi082_106363_object32665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:30.985261+0000", 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4.8072095299999997, 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.092420070000000007, 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.819 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 32666 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985261+0000", 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.820 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985261+0000", 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985259+0000", 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985293+0000", 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3232000000000003e-05 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985315+0000", 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2439e-05 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985325+0000", 2023-10-09T04:27:35.822 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7659999999999993e-06 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985376+0000", 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0788000000000002e-05 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985501+0000", 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012489900000000001 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.823 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985646+0000", 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000145344 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076882+0000", 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.091235811999999999 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076951+0000", 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9660999999999996e-05 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077566+0000", 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00061428399999999999 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077619+0000", 2023-10-09T04:27:35.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3377000000000001e-05 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077681+0000", 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2030000000000001e-05 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.826 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:32667 5.5 5:a9702beb:::benchmark_data_smithi082_106363_object32666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:30.985342+0000", 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4.8071287439999999, 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.092604212000000005, 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.827 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 32667 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985342+0000", 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985342+0000", 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985341+0000", 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.829 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985367+0000", 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6095e-05 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985377+0000", 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0210000000000001e-05 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.830 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985386+0000", 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3019999999999995e-06 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985429+0000", 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3136999999999997e-05 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985568+0000", 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013889499999999999 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985701+0000", 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013330699999999999 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076734+0000", 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.091033272999999998 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.832 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077675+0000", 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00094086500000000004 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077865+0000", 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018992999999999999 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077907+0000", 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1668000000000002e-05 2023-10-09T04:27:35.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077946+0000", 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9215999999999999e-05 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:32668 5.c e110/106 5:3215e096:::benchmark_data_smithi082_106363_object32667:head v 110'2053, mlcod=110'2051)", 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:30.985971+0000", 2023-10-09T04:27:35.834 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4.8064992850000001, 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.090854180000000007, 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985971+0000", 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985971+0000", 2023-10-09T04:27:35.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.985970+0000", 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.986005+0000", 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4786999999999997e-05 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.986016+0000", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1110999999999999e-05 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.986024+0000", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9340000000000006e-06 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.986105+0000", 2023-10-09T04:27:35.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0396999999999996e-05 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.986127+0000", 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2750000000000001e-05 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076772+0000", 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.090644988999999995 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.838 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076825+0000", 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3000000000000001e-05 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:32660 5.e e110/106 5:7353f940:::benchmark_data_smithi082_106363_object32659:head v 110'2087, mlcod=110'2084)", 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:30.987665+0000", 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4.804805247, 2023-10-09T04:27:35.839 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.095495872999999995, 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987665+0000", 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987665+0000", 2023-10-09T04:27:35.840 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987664+0000", 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987721+0000", 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6943e-05 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076913+0000", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.089191561000000003 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076923+0000", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.9040000000000004e-06 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077038+0000", 2023-10-09T04:27:35.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000115066 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.077091+0000", 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3664e-05 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.083109+0000", 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060180279999999999 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.083161+0000", 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1749999999999997e-05 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:32666 5.3 e110/106 ondisk, result = 0)", 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:30.987797+0000", 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4.8046735370000002, 2023-10-09T04:27:35.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.089158662999999999, 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987797+0000", 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987797+0000", 2023-10-09T04:27:35.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987796+0000", 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:30.987802+0000", 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0179999999999998e-06 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076871+0000", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.089069529999999994 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076882+0000", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0356e-05 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076934+0000", 2023-10-09T04:27:35.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2232999999999999e-05 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076944+0000", 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0301e-05 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:31.076956+0000", 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1249e-05 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.848 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:37425 5.3 5:c5888792:::benchmark_data_smithi082_106363_object37424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.959600+0000", 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 1.832870231, 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.073737109999999995, 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.849 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 37425 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959600+0000", 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959600+0000", 2023-10-09T04:27:35.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959599+0000", 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959625+0000", 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6078e-05 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959636+0000", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1518999999999999e-05 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959644+0000", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5299999999999999e-06 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959681+0000", 2023-10-09T04:27:35.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7116000000000002e-05 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959798+0000", 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011677599999999999 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959918+0000", 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012032999999999999 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.961290+0000", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001371347 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.962215+0000", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000925824 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.033193+0000", 2023-10-09T04:27:35.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.070977686999999998 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.033274+0000", 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0650999999999997e-05 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.033337+0000", 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3629999999999999e-05 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.855 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:37427 5.5 5:a92eabae:::benchmark_data_smithi082_106363_object37426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.959660+0000", 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 1.832810032, 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.074792047, 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.856 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 37427 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959660+0000", 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959660+0000", 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959659+0000", 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959687+0000", 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8198e-05 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959694+0000", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2570000000000004e-06 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959699+0000", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1000000000000003e-06 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959751+0000", 2023-10-09T04:27:35.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2581999999999999e-05 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959844+0000", 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2739999999999996e-05 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.959939+0000", 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4512999999999994e-05 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.962329+0000", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0023898410000000002 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.962458+0000", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012925299999999999 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.034247+0000", 2023-10-09T04:27:35.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.071788722999999999 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.034361+0000", 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000114098 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.034453+0000", 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1912999999999999e-05 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.862 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:37426 5.7 e110/106 5:e0512540:::benchmark_data_smithi082_106363_object37425:head v 110'2300, mlcod=110'2298)", 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.964547+0000", 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 1.8279231380000001, 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.072204718000000001, 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.964547+0000", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.964547+0000", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.964546+0000", 2023-10-09T04:27:35.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.964592+0000", 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5995999999999998e-05 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.032758+0000", 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.068165124999999993 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.032779+0000", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1634e-05 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.033053+0000", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027356899999999998 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.033091+0000", 2023-10-09T04:27:35.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7818000000000003e-05 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.036474+0000", 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0033838570000000001 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.036752+0000", 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027765800000000002 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.867 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38621 5.5 5:ae2b19f4:::benchmark_data_smithi082_106363_object38620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.805844+0000", 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.986626063, 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17511402600000001, 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.868 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38621 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805844+0000", 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805844+0000", 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805843+0000", 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805867+0000", 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3850999999999999e-05 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805875+0000", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1089999999999998e-06 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.805880+0000", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.045e-06 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806004+0000", 2023-10-09T04:27:35.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012468700000000001 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806129+0000", 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012428999999999999 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806229+0000", 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010018600000000001 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811017+0000", 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0047881529999999999 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811026+0000", 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5050000000000007e-06 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.873 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811138+0000", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000112478 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811170+0000", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1887999999999997e-05 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980958+0000", 2023-10-09T04:27:35.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.169788523 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38623 5.5 5:af886f1b:::benchmark_data_smithi082_106363_object38622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.807095+0000", 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98537578599999998, 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17477259000000001, 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.875 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38623 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807095+0000", 2023-10-09T04:27:35.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807095+0000", 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807094+0000", 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807119+0000", 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5106e-05 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807129+0000", 2023-10-09T04:27:35.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8570000000000007e-06 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807136+0000", 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1899999999999998e-06 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807867+0000", 2023-10-09T04:27:35.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00073130700000000005 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807960+0000", 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2870999999999999e-05 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808155+0000", 2023-10-09T04:27:35.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019516100000000001 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811062+0000", 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002906634 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981492+0000", 2023-10-09T04:27:35.881 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.170430049 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981503+0000", 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0757999999999999e-05 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981546+0000", 2023-10-09T04:27:35.882 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3087000000000003e-05 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981867+0000", 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00032155399999999999 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.883 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38626 5.3 5:c9de3a13:::benchmark_data_smithi082_106363_object38625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.809149+0000", 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98332114199999998, 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17411438700000001, 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38626 2023-10-09T04:27:35.884 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809149+0000", 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809149+0000", 2023-10-09T04:27:35.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809148+0000", 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.886 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809170+0000", 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1985999999999999e-05 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809181+0000", 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0516000000000001e-05 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.887 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809187+0000", 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2960000000000004e-06 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809366+0000", 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000178913 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809517+0000", 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000151137 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.888 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.809659+0000", 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014241300000000001 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983111+0000", 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17345142199999999 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.889 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983127+0000", 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6490999999999998e-05 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983161+0000", 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3964999999999998e-05 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983220+0000", 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8606000000000001e-05 2023-10-09T04:27:35.890 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983264+0000", 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3909000000000003e-05 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38629 5.3 5:c4c6daa0:::benchmark_data_smithi082_106363_object38628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.810063+0000", 2023-10-09T04:27:35.891 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.982407641, 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17896310900000001, 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38629 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810063+0000", 2023-10-09T04:27:35.892 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810063+0000", 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810062+0000", 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.893 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810085+0000", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3031999999999999e-05 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810094+0000", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2690000000000005e-06 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810100+0000", 2023-10-09T04:27:35.894 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5230000000000003e-06 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981234+0000", 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17113428 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981393+0000", 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000158813 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981547+0000", 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000153735 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.984241+0000", 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0026947770000000002 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.987885+0000", 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003644024 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.988940+0000", 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0010545190000000001 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.988988+0000", 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7691000000000001e-05 2023-10-09T04:27:35.897 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.989026+0000", 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8390999999999999e-05 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38621 5.5 e110/106 ondisk, result = 0)", 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.810807+0000", 2023-10-09T04:27:35.898 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98166323700000002, 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17018288000000001, 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810807+0000", 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.899 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810807+0000", 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810807+0000", 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810810+0000", 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1930000000000002e-06 2023-10-09T04:27:35.900 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810816+0000", 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4679999999999997e-06 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.810820+0000", 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9480000000000001e-06 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.901 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811129+0000", 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00030820899999999999 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811134+0000", 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.648e-06 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.902 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980990+0000", 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.16985591 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38623 5.5 e110/106 ondisk, result = 0)", 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.811083+0000", 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98138705699999995, 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17078728800000001, 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.903 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811083+0000", 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811083+0000", 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811083+0000", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811088+0000", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5389999999999998e-06 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811099+0000", 2023-10-09T04:27:35.905 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0485999999999999e-05 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811106+0000", 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.08e-06 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981478+0000", 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.170371879 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.906 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981486+0000", 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0530000000000007e-06 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981871+0000", 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00038495100000000002 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.907 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38626 5.3 e110/106 ondisk, result = 0)", 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.811189+0000", 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98128177900000002, 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17208284200000001, 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811189+0000", 2023-10-09T04:27:35.908 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811189+0000", 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811188+0000", 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.909 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.811197+0000", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4440000000000008e-06 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981182+0000", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.169985264 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981193+0000", 2023-10-09T04:27:35.910 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0781000000000001e-05 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983147+0000", 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001954604 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983156+0000", 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5429999999999994e-06 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.911 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983272+0000", 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000115758 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38626 5.3 e110/106 ondisk, result = 0)", 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.812235+0000", 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.98023544600000001, 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17088313799999999, 2023-10-09T04:27:35.912 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.812235+0000", 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.812235+0000", 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.913 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.812234+0000", 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.812239+0000", 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1969999999999998e-06 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.914 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980297+0000", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.1680584 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980314+0000", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6532999999999999e-05 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983092+0000", 2023-10-09T04:27:35.915 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027788119999999999 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983103+0000", 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1109e-05 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983118+0000", 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4705999999999999e-05 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.916 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.917 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:27:35.917 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:27:35.917 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:27:35.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:35.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:35.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:35.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:27:35.946553+0000 min lat: 0.00135884 max lat: 0.18495 avg lat: 0.00797355 2023-10-09T04:27:35.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:27:35.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 16 40117 40101 125.294 92.625 0.0162827 0.00797355 2023-10-09T04:27:35.972 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:27:36.012 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:27:36.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.048 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:27:36.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:36.085 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:27:36.121 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:27:36.154 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:27:36.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:36.195 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:27:36.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:36.233 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:27:36.247 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.248 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:27:36.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:36.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:36.473 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:36.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.589 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:36 smithi153 bash[59565]: cluster 2023-10-09T04:27:35.358948+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:36.590 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:36 smithi153 bash[59565]: audit 2023-10-09T04:27:35.931331+0000 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.82:0/317721441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:36.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:36.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:36.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:36 smithi082 bash[77063]: cluster 2023-10-09T04:27:35.358948+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:36.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:36 smithi082 bash[77063]: audit 2023-10-09T04:27:35.931331+0000 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.82:0/317721441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:36.698 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:36 smithi082 bash[79733]: cluster 2023-10-09T04:27:35.358948+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:27:36.698 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:36 smithi082 bash[79733]: audit 2023-10-09T04:27:35.931331+0000 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.82:0/317721441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:36.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:36.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:36.840 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:36.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:36.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 41462 41446 123.33 84.0625 0.00907361 0.00807228 2023-10-09T04:27:36.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:36.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:36.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:37.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.126 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:37.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:37.265 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:27:37.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:37.270 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:27:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:37.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:37.459 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:27:37.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:37.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:37.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:37.650 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:37 smithi153 bash[59565]: audit 2023-10-09T04:27:36.935698+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.82:0/1088499873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.650 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:37 smithi153 bash[59565]: audit 2023-10-09T04:27:36.935993+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:37.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:37.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:37 smithi082 bash[77063]: audit 2023-10-09T04:27:36.935698+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.82:0/1088499873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:37 smithi082 bash[77063]: audit 2023-10-09T04:27:36.935993+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:37 smithi082 bash[79733]: audit 2023-10-09T04:27:36.935698+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.82:0/1088499873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:37 smithi082 bash[79733]: audit 2023-10-09T04:27:36.935993+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:37.770 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:37.791 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:37.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:37.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:37.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:37.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 42901 42885 121.812 89.9375 0.0123239 0.00820371 2023-10-09T04:27:38.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:38.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:38.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:38.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:38.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:38.291 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:38.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:38.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:38.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:38.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:38.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:38.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:38.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:38.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: audit 2023-10-09T04:27:37.247511+0000 mgr.y (mgr.34107) 396 : audit 0 from='client.34812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:38.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.359436+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:27:38.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.429806+0000 mon.a (mon.0) 563 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:38.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.441606+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.448552+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: audit 2023-10-09T04:27:37.450099+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.450615+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.451840+0000 mon.a (mon.0) 565 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.452500+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.452604+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[77063]: cluster 2023-10-09T04:27:37.454809+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: audit 2023-10-09T04:27:37.247511+0000 mgr.y (mgr.34107) 396 : audit 0 from='client.34812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.359436+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.429806+0000 mon.a (mon.0) 563 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:38.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.441606+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.448552+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: audit 2023-10-09T04:27:37.450099+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.450615+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.451840+0000 mon.a (mon.0) 565 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.452500+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.452604+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:38 smithi082 bash[79733]: cluster 2023-10-09T04:27:37.454809+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:38.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:38.837 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: audit 2023-10-09T04:27:37.247511+0000 mgr.y (mgr.34107) 396 : audit 0 from='client.34812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:38.837 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.359436+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:27:38.837 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.429806+0000 mon.a (mon.0) 563 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.441606+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.448552+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: audit 2023-10-09T04:27:37.450099+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.450615+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.451840+0000 mon.a (mon.0) 565 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.452500+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.452604+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.838 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:38 smithi153 bash[59565]: cluster 2023-10-09T04:27:37.454809+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T04:27:38.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:38.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:27:38.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 16 44369 44353 120.505 91.75 0.00628243 0.00829147 2023-10-09T04:27:38.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:38.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:38.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:39.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:39.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:39.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:39.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:39.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:39.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:39.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:39.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:39.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:39.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:39.678 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:39.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:39.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:39.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:39.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:39.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:39.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 45877 45861 119.411 94.25 0.00703301 0.00836854 2023-10-09T04:27:40.005 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:39] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-10-09T04:27:40.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:40.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:40.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:40.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:40.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:40.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:40.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:40.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:40.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:40.570 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:40 smithi082 bash[77063]: cluster 2023-10-09T04:27:39.359902+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:27:40.571 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:40 smithi082 bash[79733]: cluster 2023-10-09T04:27:39.359902+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:27:40.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:40.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:40.633 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":332,"stamp":"2023-10-09T04:27:35.358462+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2403538867,"num_objects":36630,"num_object_clones":0,"num_object_copies":109890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36630,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":37708,"num_write_kb":2357687,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37772,"ondisk_log_size":37772,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8394132,"kb_used_data":7021140,"kb_used_omap":34,"kb_used_meta":1372893,"kb_avail":741567084,"statfs":{"total":767960285184,"available":759364694016,"internally_reserved":0,"allocated":7189647360,"data_stored":7185489064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1405843157},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,3,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1551302656,"num_objects":23671,"num_object_clones":0,"num_object_copies":71013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23671,"num_write_kb":1514944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003527"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930477+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:27:11.930477+0000","last_peered":"2023-10-09T04:27:11.930477+0000","last_clean":"2023-10-09T04:27:11.930477+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:27:11.930477+0000","last_undegraded":"2023-10-09T04:27:11.930477+0000","last_fullsized":"2023-10-09T04:27:11.930477+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.438129+0000","last_change":"2023-10-09T04:24:00.826023+0000","last_active":"2023-10-09T04:27:11.438129+0000","last_peered":"2023-10-09T04:27:11.438129+0000","last_clean":"2023-10-09T04:27:11.438129+0000","last_became_active":"2023-10-09T04:24:00.825466+0000","last_became_peered":"2023-10-09T04:24:00.825466+0000","last_unstale":"2023-10-09T04:27:11.438129+0000","last_undegraded":"2023-10-09T04:27:11.438129+0000","last_fullsized":"2023-10-09T04:27:11.438129+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:39.932991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622288+0000","last_change":"2023-10-09T04:24:00.825647+0000","last_active":"2023-10-09T04:27:11.622288+0000","last_peered":"2023-10-09T04:27:11.622288+0000","last_clean":"2023-10-09T04:27:11.622288+0000","last_became_active":"2023-10-09T04:24:00.825310+0000","last_became_peered":"2023-10-09T04:24:00.825310+0000","last_unstale":"2023-10-09T04:27:11.622288+0000","last_undegraded":"2023-10-09T04:27:11.622288+0000","last_fullsized":"2023-10-09T04:27:11.622288+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:07.180568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202443+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:27:12.202443+0000","last_peered":"2023-10-09T04:27:12.202443+0000","last_clean":"2023-10-09T04:27:12.202443+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:27:12.202443+0000","last_undegraded":"2023-10-09T04:27:12.202443+0000","last_fullsized":"2023-10-09T04:27:12.202443+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":96,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437302+0000","last_change":"2023-10-09T04:24:00.814141+0000","last_active":"2023-10-09T04:27:11.437302+0000","last_peered":"2023-10-09T04:27:11.437302+0000","last_clean":"2023-10-09T04:27:11.437302+0000","last_became_active":"2023-10-09T04:24:00.813799+0000","last_became_peered":"2023-10-09T04:24:00.813799+0000","last_unstale":"2023-10-09T04:27:11.437302+0000","last_undegraded":"2023-10-09T04:27:11.437302+0000","last_fullsized":"2023-10-09T04:27:11.437302+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:37.871798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440583+0000","last_change":"2023-10-09T04:22:50.702787+0000","last_active":"2023-10-09T04:27:11.440583+0000","last_peered":"2023-10-09T04:27:11.440583+0000","last_clean":"2023-10-09T04:27:11.440583+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:22:50.702392+0000","last_unstale":"2023-10-09T04:27:11.440583+0000","last_undegraded":"2023-10-09T04:27:11.440583+0000","last_fullsized":"2023-10-09T04:27:11.440583+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202845+0000","last_change":"2023-10-09T04:24:00.816214+0000","last_active":"2023-10-09T04:27:12.202845+0000","last_peered":"2023-10-09T04:27:12.202845+0000","last_clean":"2023-10-09T04:27:12.202845+0000","last_became_active":"2023-10-09T04:24:00.815867+0000","last_became_peered":"2023-10-09T04:24:00.815867+0000","last_unstale":"2023-10-09T04:27:12.202845+0000","last_undegraded":"2023-10-09T04:27:12.202845+0000","last_fullsized":"2023-10-09T04:27:12.202845+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:24.219649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437194+0000","last_change":"2023-10-09T04:24:00.820037+0000","last_active":"2023-10-09T04:27:11.437194+0000","last_peered":"2023-10-09T04:27:11.437194+0000","last_clean":"2023-10-09T04:27:11.437194+0000","last_became_active":"2023-10-09T04:24:00.819650+0000","last_became_peered":"2023-10-09T04:24:00.819650+0000","last_unstale":"2023-10-09T04:27:11.437194+0000","last_undegraded":"2023-10-09T04:27:11.437194+0000","last_fullsized":"2023-10-09T04:27:11.437194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:32.924825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959618+0000","last_change":"2023-10-09T04:22:50.698679+0000","last_active":"2023-10-09T04:27:11.959618+0000","last_peered":"2023-10-09T04:27:11.959618+0000","last_clean":"2023-10-09T04:27:11.959618+0000","last_became_active":"2023-10-09T04:22:50.698360+0000","last_became_peered":"2023-10-09T04:22:50.698360+0000","last_unstale":"2023-10-09T04:27:11.959618+0000","last_undegraded":"2023-10-09T04:27:11.959618+0000","last_fullsized":"2023-10-09T04:27:11.959618+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437897+0000","last_change":"2023-10-09T04:24:00.822436+0000","last_active":"2023-10-09T04:27:11.437897+0000","last_peered":"2023-10-09T04:27:11.437897+0000","last_clean":"2023-10-09T04:27:11.437897+0000","last_became_active":"2023-10-09T04:24:00.822108+0000","last_became_peered":"2023-10-09T04:24:00.822108+0000","last_unstale":"2023-10-09T04:27:11.437897+0000","last_undegraded":"2023-10-09T04:27:11.437897+0000","last_fullsized":"2023-10-09T04:27:11.437897+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:24.737042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492783+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:27:11.492783+0000","last_peered":"2023-10-09T04:27:11.492783+0000","last_clean":"2023-10-09T04:27:11.492783+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:27:11.492783+0000","last_undegraded":"2023-10-09T04:27:11.492783+0000","last_fullsized":"2023-10-09T04:27:11.492783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:13.531258+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:27:13.531258+0000","last_peered":"2023-10-09T04:27:13.531258+0000","last_clean":"2023-10-09T04:27:13.531258+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:27:13.531258+0000","last_undegraded":"2023-10-09T04:27:13.531258+0000","last_fullsized":"2023-10-09T04:27:13.531258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931000+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:27:11.931000+0000","last_peered":"2023-10-09T04:27:11.931000+0000","last_clean":"2023-10-09T04:27:11.931000+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:27:11.931000+0000","last_undegraded":"2023-10-09T04:27:11.931000+0000","last_fullsized":"2023-10-09T04:27:11.931000+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931128+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:27:11.931128+0000","last_peered":"2023-10-09T04:27:11.931128+0000","last_clean":"2023-10-09T04:27:11.931128+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:27:11.931128+0000","last_undegraded":"2023-10-09T04:27:11.931128+0000","last_fullsized":"2023-10-09T04:27:11.931128+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492322+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:27:11.492322+0000","last_peered":"2023-10-09T04:27:11.492322+0000","last_clean":"2023-10-09T04:27:11.492322+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:27:11.492322+0000","last_undegraded":"2023-10-09T04:27:11.492322+0000","last_fullsized":"2023-10-09T04:27:11.492322+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437891+0000","last_change":"2023-10-09T04:24:00.821914+0000","last_active":"2023-10-09T04:27:11.437891+0000","last_peered":"2023-10-09T04:27:11.437891+0000","last_clean":"2023-10-09T04:27:11.437891+0000","last_became_active":"2023-10-09T04:24:00.821588+0000","last_became_peered":"2023-10-09T04:24:00.821588+0000","last_unstale":"2023-10-09T04:27:11.437891+0000","last_undegraded":"2023-10-09T04:27:11.437891+0000","last_fullsized":"2023-10-09T04:27:11.437891+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:52.558769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202218+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:27:12.202218+0000","last_peered":"2023-10-09T04:27:12.202218+0000","last_clean":"2023-10-09T04:27:12.202218+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:27:12.202218+0000","last_undegraded":"2023-10-09T04:27:12.202218+0000","last_fullsized":"2023-10-09T04:27:12.202218+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959501+0000","last_change":"2023-10-09T04:24:00.822638+0000","last_active":"2023-10-09T04:27:11.959501+0000","last_peered":"2023-10-09T04:27:11.959501+0000","last_clean":"2023-10-09T04:27:11.959501+0000","last_became_active":"2023-10-09T04:24:00.822103+0000","last_became_peered":"2023-10-09T04:24:00.822103+0000","last_unstale":"2023-10-09T04:27:11.959501+0000","last_undegraded":"2023-10-09T04:27:11.959501+0000","last_fullsized":"2023-10-09T04:27:11.959501+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492344+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:27:11.492344+0000","last_peered":"2023-10-09T04:27:11.492344+0000","last_clean":"2023-10-09T04:27:11.492344+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:27:11.492344+0000","last_undegraded":"2023-10-09T04:27:11.492344+0000","last_fullsized":"2023-10-09T04:27:11.492344+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202368+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:27:12.202368+0000","last_peered":"2023-10-09T04:27:12.202368+0000","last_clean":"2023-10-09T04:27:12.202368+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:27:12.202368+0000","last_undegraded":"2023-10-09T04:27:12.202368+0000","last_fullsized":"2023-10-09T04:27:12.202368+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440226+0000","last_change":"2023-10-09T04:22:23.612613+0000","last_active":"2023-10-09T04:27:11.440226+0000","last_peered":"2023-10-09T04:27:11.440226+0000","last_clean":"2023-10-09T04:27:11.440226+0000","last_became_active":"2023-10-09T04:22:23.611940+0000","last_became_peered":"2023-10-09T04:22:23.611940+0000","last_unstale":"2023-10-09T04:27:11.440226+0000","last_undegraded":"2023-10-09T04:27:11.440226+0000","last_fullsized":"2023-10-09T04:27:11.440226+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.438440+0000","last_change":"2023-10-09T04:24:00.822512+0000","last_active":"2023-10-09T04:27:11.438440+0000","last_peered":"2023-10-09T04:27:11.438440+0000","last_clean":"2023-10-09T04:27:11.438440+0000","last_became_active":"2023-10-09T04:24:00.822183+0000","last_became_peered":"2023-10-09T04:24:00.822183+0000","last_unstale":"2023-10-09T04:27:11.438440+0000","last_undegraded":"2023-10-09T04:27:11.438440+0000","last_fullsized":"2023-10-09T04:27:11.438440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:42.029015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"110'2482","reported_seq":2494,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.156228+0000","last_change":"2023-10-09T04:26:55.440461+0000","last_active":"2023-10-09T04:27:35.156228+0000","last_peered":"2023-10-09T04:27:35.156228+0000","last_clean":"2023-10-09T04:27:35.156228+0000","last_became_active":"2023-10-09T04:26:55.440099+0000","last_became_peered":"2023-10-09T04:26:55.440099+0000","last_unstale":"2023-10-09T04:27:35.156228+0000","last_undegraded":"2023-10-09T04:27:35.156228+0000","last_fullsized":"2023-10-09T04:27:35.156228+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2482,"log_dups_size":0,"ondisk_log_size":2482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162660352,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":158848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.493083+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:27:11.493083+0000","last_peered":"2023-10-09T04:27:11.493083+0000","last_clean":"2023-10-09T04:27:11.493083+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:27:11.493083+0000","last_undegraded":"2023-10-09T04:27:11.493083+0000","last_fullsized":"2023-10-09T04:27:11.493083+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.958751+0000","last_change":"2023-10-09T04:11:27.724252+0000","last_active":"2023-10-09T04:27:11.958751+0000","last_peered":"2023-10-09T04:27:11.958751+0000","last_clean":"2023-10-09T04:27:11.958751+0000","last_became_active":"2023-10-09T04:11:27.723509+0000","last_became_peered":"2023-10-09T04:11:27.723509+0000","last_unstale":"2023-10-09T04:27:11.958751+0000","last_undegraded":"2023-10-09T04:27:11.958751+0000","last_fullsized":"2023-10-09T04:27:11.958751+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"110'2155","reported_seq":2168,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.779115+0000","last_change":"2023-10-09T04:26:55.433159+0000","last_active":"2023-10-09T04:27:31.779115+0000","last_peered":"2023-10-09T04:27:31.779115+0000","last_clean":"2023-10-09T04:27:31.779115+0000","last_became_active":"2023-10-09T04:26:55.432784+0000","last_became_peered":"2023-10-09T04:26:55.432784+0000","last_unstale":"2023-10-09T04:27:31.779115+0000","last_undegraded":"2023-10-09T04:27:31.779115+0000","last_fullsized":"2023-10-09T04:27:31.779115+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2155,"log_dups_size":0,"ondisk_log_size":2155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:19.007979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141230080,"num_objects":2155,"num_object_clones":0,"num_object_copies":6465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2155,"num_write_kb":137920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492960+0000","last_change":"2023-10-09T04:22:50.692658+0000","last_active":"2023-10-09T04:27:11.492960+0000","last_peered":"2023-10-09T04:27:11.492960+0000","last_clean":"2023-10-09T04:27:11.492960+0000","last_became_active":"2023-10-09T04:22:50.692027+0000","last_became_peered":"2023-10-09T04:22:50.692027+0000","last_unstale":"2023-10-09T04:27:11.492960+0000","last_undegraded":"2023-10-09T04:27:11.492960+0000","last_fullsized":"2023-10-09T04:27:11.492960+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930600+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:27:11.930600+0000","last_peered":"2023-10-09T04:27:11.930600+0000","last_clean":"2023-10-09T04:27:11.930600+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:27:11.930600+0000","last_undegraded":"2023-10-09T04:27:11.930600+0000","last_fullsized":"2023-10-09T04:27:11.930600+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"110'2135","reported_seq":2148,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.760763+0000","last_change":"2023-10-09T04:26:55.436643+0000","last_active":"2023-10-09T04:27:31.760763+0000","last_peered":"2023-10-09T04:27:31.760763+0000","last_clean":"2023-10-09T04:27:31.760763+0000","last_became_active":"2023-10-09T04:26:55.436302+0000","last_became_peered":"2023-10-09T04:26:55.436302+0000","last_unstale":"2023-10-09T04:27:31.760763+0000","last_undegraded":"2023-10-09T04:27:31.760763+0000","last_fullsized":"2023-10-09T04:27:31.760763+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2135,"log_dups_size":0,"ondisk_log_size":2135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:55.643933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139919360,"num_objects":2135,"num_object_clones":0,"num_object_copies":6405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2135,"num_write_kb":136640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492928+0000","last_change":"2023-10-09T04:11:26.215181+0000","last_active":"2023-10-09T04:27:11.492928+0000","last_peered":"2023-10-09T04:27:11.492928+0000","last_clean":"2023-10-09T04:27:11.492928+0000","last_became_active":"2023-10-09T04:11:26.214874+0000","last_became_peered":"2023-10-09T04:11:26.214874+0000","last_unstale":"2023-10-09T04:27:11.492928+0000","last_undegraded":"2023-10-09T04:27:11.492928+0000","last_fullsized":"2023-10-09T04:27:11.492928+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440086+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:27:11.440086+0000","last_peered":"2023-10-09T04:27:11.440086+0000","last_clean":"2023-10-09T04:27:11.440086+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:27:11.440086+0000","last_undegraded":"2023-10-09T04:27:11.440086+0000","last_fullsized":"2023-10-09T04:27:11.440086+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"110'2113","reported_seq":2126,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.787079+0000","last_change":"2023-10-09T04:26:55.435227+0000","last_active":"2023-10-09T04:27:31.787079+0000","last_peered":"2023-10-09T04:27:31.787079+0000","last_clean":"2023-10-09T04:27:31.787079+0000","last_became_active":"2023-10-09T04:26:55.434919+0000","last_became_peered":"2023-10-09T04:26:55.434919+0000","last_unstale":"2023-10-09T04:27:31.787079+0000","last_undegraded":"2023-10-09T04:27:31.787079+0000","last_fullsized":"2023-10-09T04:27:31.787079+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2113,"log_dups_size":0,"ondisk_log_size":2113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:38.324241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138477568,"num_objects":2113,"num_object_clones":0,"num_object_copies":6339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2113,"num_write_kb":135232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959088+0000","last_change":"2023-10-09T04:24:00.822324+0000","last_active":"2023-10-09T04:27:11.959088+0000","last_peered":"2023-10-09T04:27:11.959088+0000","last_clean":"2023-10-09T04:27:11.959088+0000","last_became_active":"2023-10-09T04:24:00.821777+0000","last_became_peered":"2023-10-09T04:24:00.821777+0000","last_unstale":"2023-10-09T04:27:11.959088+0000","last_undegraded":"2023-10-09T04:27:11.959088+0000","last_fullsized":"2023-10-09T04:27:11.959088+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202250+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:27:12.202250+0000","last_peered":"2023-10-09T04:27:12.202250+0000","last_clean":"2023-10-09T04:27:12.202250+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:27:12.202250+0000","last_undegraded":"2023-10-09T04:27:12.202250+0000","last_fullsized":"2023-10-09T04:27:12.202250+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"110'2161","reported_seq":2172,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.761201+0000","last_change":"2023-10-09T04:26:55.436327+0000","last_active":"2023-10-09T04:27:31.761201+0000","last_peered":"2023-10-09T04:27:31.761201+0000","last_clean":"2023-10-09T04:27:31.761201+0000","last_became_active":"2023-10-09T04:26:55.435991+0000","last_became_peered":"2023-10-09T04:26:55.435991+0000","last_unstale":"2023-10-09T04:27:31.761201+0000","last_undegraded":"2023-10-09T04:27:31.761201+0000","last_fullsized":"2023-10-09T04:27:31.761201+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2161,"log_dups_size":0,"ondisk_log_size":2161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:58.658715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141623296,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2161,"num_write_kb":138304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437139+0000","last_change":"2023-10-09T04:24:00.826876+0000","last_active":"2023-10-09T04:27:11.437139+0000","last_peered":"2023-10-09T04:27:11.437139+0000","last_clean":"2023-10-09T04:27:11.437139+0000","last_became_active":"2023-10-09T04:24:00.826469+0000","last_became_peered":"2023-10-09T04:24:00.826469+0000","last_unstale":"2023-10-09T04:27:11.437139+0000","last_undegraded":"2023-10-09T04:27:11.437139+0000","last_fullsized":"2023-10-09T04:27:11.437139+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:08.758269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931198+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:27:11.931198+0000","last_peered":"2023-10-09T04:27:11.931198+0000","last_clean":"2023-10-09T04:27:11.931198+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:27:11.931198+0000","last_undegraded":"2023-10-09T04:27:11.931198+0000","last_fullsized":"2023-10-09T04:27:11.931198+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"110'2342","reported_seq":2355,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:33.679474+0000","last_change":"2023-10-09T04:26:55.433498+0000","last_active":"2023-10-09T04:27:33.679474+0000","last_peered":"2023-10-09T04:27:33.679474+0000","last_clean":"2023-10-09T04:27:33.679474+0000","last_became_active":"2023-10-09T04:26:55.433162+0000","last_became_peered":"2023-10-09T04:26:55.433162+0000","last_unstale":"2023-10-09T04:27:33.679474+0000","last_undegraded":"2023-10-09T04:27:33.679474+0000","last_fullsized":"2023-10-09T04:27:33.679474+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2342,"log_dups_size":0,"ondisk_log_size":2342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:56.313689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153485312,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2342,"num_write_kb":149888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440298+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:27:11.440298+0000","last_peered":"2023-10-09T04:27:11.440298+0000","last_clean":"2023-10-09T04:27:11.440298+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:27:11.440298+0000","last_undegraded":"2023-10-09T04:27:11.440298+0000","last_fullsized":"2023-10-09T04:27:11.440298+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931174+0000","last_change":"2023-10-09T04:23:18.773293+0000","last_active":"2023-10-09T04:27:11.931174+0000","last_peered":"2023-10-09T04:27:11.931174+0000","last_clean":"2023-10-09T04:27:11.931174+0000","last_became_active":"2023-10-09T04:23:18.772947+0000","last_became_peered":"2023-10-09T04:23:18.772947+0000","last_unstale":"2023-10-09T04:27:11.931174+0000","last_undegraded":"2023-10-09T04:27:11.931174+0000","last_fullsized":"2023-10-09T04:27:11.931174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"110'2400","reported_seq":2412,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.196352+0000","last_change":"2023-10-09T04:26:55.437859+0000","last_active":"2023-10-09T04:27:35.196352+0000","last_peered":"2023-10-09T04:27:35.196352+0000","last_clean":"2023-10-09T04:27:35.196352+0000","last_became_active":"2023-10-09T04:26:55.437526+0000","last_became_peered":"2023-10-09T04:26:55.437526+0000","last_unstale":"2023-10-09T04:27:35.196352+0000","last_undegraded":"2023-10-09T04:27:35.196352+0000","last_fullsized":"2023-10-09T04:27:35.196352+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2400,"log_dups_size":0,"ondisk_log_size":2400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2400,"num_write_kb":153600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930148+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:27:11.930148+0000","last_peered":"2023-10-09T04:27:11.930148+0000","last_clean":"2023-10-09T04:27:11.930148+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:27:11.930148+0000","last_undegraded":"2023-10-09T04:27:11.930148+0000","last_fullsized":"2023-10-09T04:27:11.930148+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931539+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:27:11.931539+0000","last_peered":"2023-10-09T04:27:11.931539+0000","last_clean":"2023-10-09T04:27:11.931539+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:27:11.931539+0000","last_undegraded":"2023-10-09T04:27:11.931539+0000","last_fullsized":"2023-10-09T04:27:11.931539+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"110'2472","reported_seq":2484,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.186068+0000","last_change":"2023-10-09T04:26:55.437729+0000","last_active":"2023-10-09T04:27:35.186068+0000","last_peered":"2023-10-09T04:27:35.186068+0000","last_clean":"2023-10-09T04:27:35.186068+0000","last_became_active":"2023-10-09T04:26:55.437319+0000","last_became_peered":"2023-10-09T04:26:55.437319+0000","last_unstale":"2023-10-09T04:27:35.186068+0000","last_undegraded":"2023-10-09T04:27:35.186068+0000","last_fullsized":"2023-10-09T04:27:35.186068+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2472,"log_dups_size":0,"ondisk_log_size":2472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162004992,"num_objects":2472,"num_object_clones":0,"num_object_copies":7416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2472,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437496+0000","last_change":"2023-10-09T04:24:00.822985+0000","last_active":"2023-10-09T04:27:11.437496+0000","last_peered":"2023-10-09T04:27:11.437496+0000","last_clean":"2023-10-09T04:27:11.437496+0000","last_became_active":"2023-10-09T04:24:00.822621+0000","last_became_peered":"2023-10-09T04:24:00.822621+0000","last_unstale":"2023-10-09T04:27:11.437496+0000","last_undegraded":"2023-10-09T04:27:11.437496+0000","last_fullsized":"2023-10-09T04:27:11.437496+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:59.227822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:13.531820+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:27:13.531820+0000","last_peered":"2023-10-09T04:27:13.531820+0000","last_clean":"2023-10-09T04:27:13.531820+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:27:13.531820+0000","last_undegraded":"2023-10-09T04:27:13.531820+0000","last_fullsized":"2023-10-09T04:27:13.531820+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":480,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622463+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:27:11.622463+0000","last_peered":"2023-10-09T04:27:11.622463+0000","last_clean":"2023-10-09T04:27:11.622463+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:27:11.622463+0000","last_undegraded":"2023-10-09T04:27:11.622463+0000","last_fullsized":"2023-10-09T04:27:11.622463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959548+0000","last_change":"2023-10-09T04:24:00.813604+0000","last_active":"2023-10-09T04:27:11.959548+0000","last_peered":"2023-10-09T04:27:11.959548+0000","last_clean":"2023-10-09T04:27:11.959548+0000","last_became_active":"2023-10-09T04:24:00.813231+0000","last_became_peered":"2023-10-09T04:24:00.813231+0000","last_unstale":"2023-10-09T04:27:11.959548+0000","last_undegraded":"2023-10-09T04:27:11.959548+0000","last_fullsized":"2023-10-09T04:27:11.959548+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622579+0000","last_change":"2023-10-09T04:22:50.703397+0000","last_active":"2023-10-09T04:27:11.622579+0000","last_peered":"2023-10-09T04:27:11.622579+0000","last_clean":"2023-10-09T04:27:11.622579+0000","last_became_active":"2023-10-09T04:22:50.700260+0000","last_became_peered":"2023-10-09T04:22:50.700260+0000","last_unstale":"2023-10-09T04:27:11.622579+0000","last_undegraded":"2023-10-09T04:27:11.622579+0000","last_fullsized":"2023-10-09T04:27:11.622579+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:09.013765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"110'2047","reported_seq":2060,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.623203+0000","last_change":"2023-10-09T04:26:55.439658+0000","last_active":"2023-10-09T04:27:31.623203+0000","last_peered":"2023-10-09T04:27:31.623203+0000","last_clean":"2023-10-09T04:27:31.623203+0000","last_became_active":"2023-10-09T04:26:55.439336+0000","last_became_peered":"2023-10-09T04:26:55.439336+0000","last_unstale":"2023-10-09T04:27:31.623203+0000","last_undegraded":"2023-10-09T04:27:31.623203+0000","last_fullsized":"2023-10-09T04:27:31.623203+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2047,"log_dups_size":0,"ondisk_log_size":2047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:35.705528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134152192,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2047,"num_write_kb":131008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"110'2144","reported_seq":2157,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.643607+0000","last_change":"2023-10-09T04:26:55.434004+0000","last_active":"2023-10-09T04:27:31.643607+0000","last_peered":"2023-10-09T04:27:31.643607+0000","last_clean":"2023-10-09T04:27:31.643607+0000","last_became_active":"2023-10-09T04:26:55.433584+0000","last_became_peered":"2023-10-09T04:26:55.433584+0000","last_unstale":"2023-10-09T04:27:31.643607+0000","last_undegraded":"2023-10-09T04:27:31.643607+0000","last_fullsized":"2023-10-09T04:27:31.643607+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2144,"log_dups_size":0,"ondisk_log_size":2144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:31.621310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2144,"num_write_kb":137216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":616,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959058+0000","last_change":"2023-10-09T04:22:50.705195+0000","last_active":"2023-10-09T04:27:11.959058+0000","last_peered":"2023-10-09T04:27:11.959058+0000","last_clean":"2023-10-09T04:27:11.959058+0000","last_became_active":"2023-10-09T04:22:50.704574+0000","last_became_peered":"2023-10-09T04:22:50.704574+0000","last_unstale":"2023-10-09T04:27:11.959058+0000","last_undegraded":"2023-10-09T04:27:11.959058+0000","last_fullsized":"2023-10-09T04:27:11.959058+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.929739+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:27:11.929739+0000","last_peered":"2023-10-09T04:27:11.929739+0000","last_clean":"2023-10-09T04:27:11.929739+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:27:11.929739+0000","last_undegraded":"2023-10-09T04:27:11.929739+0000","last_fullsized":"2023-10-09T04:27:11.929739+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622598+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:27:11.622598+0000","last_peered":"2023-10-09T04:27:11.622598+0000","last_clean":"2023-10-09T04:27:11.622598+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:27:11.622598+0000","last_undegraded":"2023-10-09T04:27:11.622598+0000","last_fullsized":"2023-10-09T04:27:11.622598+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"110'2163","reported_seq":2176,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.772867+0000","last_change":"2023-10-09T04:26:55.436387+0000","last_active":"2023-10-09T04:27:31.772867+0000","last_peered":"2023-10-09T04:27:31.772867+0000","last_clean":"2023-10-09T04:27:31.772867+0000","last_became_active":"2023-10-09T04:26:55.435976+0000","last_became_peered":"2023-10-09T04:26:55.435976+0000","last_unstale":"2023-10-09T04:27:31.772867+0000","last_undegraded":"2023-10-09T04:27:31.772867+0000","last_fullsized":"2023-10-09T04:27:31.772867+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2163,"log_dups_size":0,"ondisk_log_size":2163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:23.156005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141754368,"num_objects":2163,"num_object_clones":0,"num_object_copies":6489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2163,"num_write_kb":138432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1216,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:13.442317+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:27:13.442317+0000","last_peered":"2023-10-09T04:27:13.442317+0000","last_clean":"2023-10-09T04:27:13.442317+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:27:13.442317+0000","last_undegraded":"2023-10-09T04:27:13.442317+0000","last_fullsized":"2023-10-09T04:27:13.442317+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622574+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:27:11.622574+0000","last_peered":"2023-10-09T04:27:11.622574+0000","last_clean":"2023-10-09T04:27:11.622574+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:27:11.622574+0000","last_undegraded":"2023-10-09T04:27:11.622574+0000","last_fullsized":"2023-10-09T04:27:11.622574+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.623021+0000","last_change":"2023-10-09T04:22:50.696439+0000","last_active":"2023-10-09T04:27:11.623021+0000","last_peered":"2023-10-09T04:27:11.623021+0000","last_clean":"2023-10-09T04:27:11.623021+0000","last_became_active":"2023-10-09T04:22:50.696084+0000","last_became_peered":"2023-10-09T04:22:50.696084+0000","last_unstale":"2023-10-09T04:27:11.623021+0000","last_undegraded":"2023-10-09T04:27:11.623021+0000","last_fullsized":"2023-10-09T04:27:11.623021+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.931059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":593,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.958918+0000","last_change":"2023-10-09T04:23:18.767411+0000","last_active":"2023-10-09T04:27:11.958918+0000","last_peered":"2023-10-09T04:27:11.958918+0000","last_clean":"2023-10-09T04:27:11.958918+0000","last_became_active":"2023-10-09T04:23:18.766728+0000","last_became_peered":"2023-10-09T04:23:18.766728+0000","last_unstale":"2023-10-09T04:27:11.958918+0000","last_undegraded":"2023-10-09T04:27:11.958918+0000","last_fullsized":"2023-10-09T04:27:11.958918+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":770,"num_read_kb":1698,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":312,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492335+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:27:11.492335+0000","last_peered":"2023-10-09T04:27:11.492335+0000","last_clean":"2023-10-09T04:27:11.492335+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:27:11.492335+0000","last_undegraded":"2023-10-09T04:27:11.492335+0000","last_fullsized":"2023-10-09T04:27:11.492335+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437060+0000","last_change":"2023-10-09T04:24:00.819863+0000","last_active":"2023-10-09T04:27:11.437060+0000","last_peered":"2023-10-09T04:27:11.437060+0000","last_clean":"2023-10-09T04:27:11.437060+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:24:00.819513+0000","last_unstale":"2023-10-09T04:27:11.437060+0000","last_undegraded":"2023-10-09T04:27:11.437060+0000","last_fullsized":"2023-10-09T04:27:11.437060+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:12.187156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440470+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:27:11.440470+0000","last_peered":"2023-10-09T04:27:11.440470+0000","last_clean":"2023-10-09T04:27:11.440470+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:27:11.440470+0000","last_undegraded":"2023-10-09T04:27:11.440470+0000","last_fullsized":"2023-10-09T04:27:11.440470+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"110'2423","reported_seq":2435,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.194895+0000","last_change":"2023-10-09T04:26:55.433932+0000","last_active":"2023-10-09T04:27:35.194895+0000","last_peered":"2023-10-09T04:27:35.194895+0000","last_clean":"2023-10-09T04:27:35.194895+0000","last_became_active":"2023-10-09T04:26:55.433501+0000","last_became_peered":"2023-10-09T04:26:55.433501+0000","last_unstale":"2023-10-09T04:27:35.194895+0000","last_undegraded":"2023-10-09T04:27:35.194895+0000","last_fullsized":"2023-10-09T04:27:35.194895+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2423,"log_dups_size":0,"ondisk_log_size":2423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2423,"num_write_kb":155072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"110'2421","reported_seq":2433,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.185007+0000","last_change":"2023-10-09T04:26:55.433400+0000","last_active":"2023-10-09T04:27:35.185007+0000","last_peered":"2023-10-09T04:27:35.185007+0000","last_clean":"2023-10-09T04:27:35.185007+0000","last_became_active":"2023-10-09T04:26:55.432958+0000","last_became_peered":"2023-10-09T04:26:55.432958+0000","last_unstale":"2023-10-09T04:27:35.185007+0000","last_undegraded":"2023-10-09T04:27:35.185007+0000","last_fullsized":"2023-10-09T04:27:35.185007+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2421,"log_dups_size":0,"ondisk_log_size":2421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2421,"num_write_kb":154944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":654,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492533+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:27:11.492533+0000","last_peered":"2023-10-09T04:27:11.492533+0000","last_clean":"2023-10-09T04:27:11.492533+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:27:11.492533+0000","last_undegraded":"2023-10-09T04:27:11.492533+0000","last_fullsized":"2023-10-09T04:27:11.492533+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.201674+0000","last_change":"2023-10-09T04:24:00.823054+0000","last_active":"2023-10-09T04:27:12.201674+0000","last_peered":"2023-10-09T04:27:12.201674+0000","last_clean":"2023-10-09T04:27:12.201674+0000","last_became_active":"2023-10-09T04:24:00.822733+0000","last_became_peered":"2023-10-09T04:24:00.822733+0000","last_unstale":"2023-10-09T04:27:12.201674+0000","last_undegraded":"2023-10-09T04:27:12.201674+0000","last_fullsized":"2023-10-09T04:27:12.201674+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:13.369631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202608+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:27:12.202608+0000","last_peered":"2023-10-09T04:27:12.202608+0000","last_clean":"2023-10-09T04:27:12.202608+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:27:12.202608+0000","last_undegraded":"2023-10-09T04:27:12.202608+0000","last_fullsized":"2023-10-09T04:27:12.202608+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":323,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.492906+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:27:11.492906+0000","last_peered":"2023-10-09T04:27:11.492906+0000","last_clean":"2023-10-09T04:27:11.492906+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:27:11.492906+0000","last_undegraded":"2023-10-09T04:27:11.492906+0000","last_fullsized":"2023-10-09T04:27:11.492906+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.622631+0000","last_change":"2023-10-09T04:23:18.773712+0000","last_active":"2023-10-09T04:27:11.622631+0000","last_peered":"2023-10-09T04:27:11.622631+0000","last_clean":"2023-10-09T04:27:11.622631+0000","last_became_active":"2023-10-09T04:23:18.773331+0000","last_became_peered":"2023-10-09T04:23:18.773331+0000","last_unstale":"2023-10-09T04:27:11.622631+0000","last_undegraded":"2023-10-09T04:27:11.622631+0000","last_fullsized":"2023-10-09T04:27:11.622631+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:53.528302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.201427+0000","last_change":"2023-10-09T04:22:23.611501+0000","last_active":"2023-10-09T04:27:12.201427+0000","last_peered":"2023-10-09T04:27:12.201427+0000","last_clean":"2023-10-09T04:27:12.201427+0000","last_became_active":"2023-10-09T04:22:23.611116+0000","last_became_peered":"2023-10-09T04:22:23.611116+0000","last_unstale":"2023-10-09T04:27:12.201427+0000","last_undegraded":"2023-10-09T04:27:12.201427+0000","last_fullsized":"2023-10-09T04:27:12.201427+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:19.031531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"110'2457","reported_seq":2469,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.185350+0000","last_change":"2023-10-09T04:26:55.442950+0000","last_active":"2023-10-09T04:27:35.185350+0000","last_peered":"2023-10-09T04:27:35.185350+0000","last_clean":"2023-10-09T04:27:35.185350+0000","last_became_active":"2023-10-09T04:26:55.442575+0000","last_became_peered":"2023-10-09T04:26:55.442575+0000","last_unstale":"2023-10-09T04:27:35.185350+0000","last_undegraded":"2023-10-09T04:27:35.185350+0000","last_fullsized":"2023-10-09T04:27:35.185350+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2457,"log_dups_size":0,"ondisk_log_size":2457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161021952,"num_objects":2457,"num_object_clones":0,"num_object_copies":7371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2457,"num_write_kb":157248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.438566+0000","last_change":"2023-10-09T04:24:00.824546+0000","last_active":"2023-10-09T04:27:11.438566+0000","last_peered":"2023-10-09T04:27:11.438566+0000","last_clean":"2023-10-09T04:27:11.438566+0000","last_became_active":"2023-10-09T04:24:00.824005+0000","last_became_peered":"2023-10-09T04:24:00.824005+0000","last_unstale":"2023-10-09T04:27:11.438566+0000","last_undegraded":"2023-10-09T04:27:11.438566+0000","last_fullsized":"2023-10-09T04:27:11.438566+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:46.407835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":331,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.623020+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:27:11.623020+0000","last_peered":"2023-10-09T04:27:11.623020+0000","last_clean":"2023-10-09T04:27:11.623020+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:27:11.623020+0000","last_undegraded":"2023-10-09T04:27:11.623020+0000","last_fullsized":"2023-10-09T04:27:11.623020+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437398+0000","last_change":"2023-10-09T04:24:00.822119+0000","last_active":"2023-10-09T04:27:11.437398+0000","last_peered":"2023-10-09T04:27:11.437398+0000","last_clean":"2023-10-09T04:27:11.437398+0000","last_became_active":"2023-10-09T04:24:00.821789+0000","last_became_peered":"2023-10-09T04:24:00.821789+0000","last_unstale":"2023-10-09T04:27:11.437398+0000","last_undegraded":"2023-10-09T04:27:11.437398+0000","last_fullsized":"2023-10-09T04:27:11.437398+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:09.080969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"110'2418","reported_seq":2430,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.180713+0000","last_change":"2023-10-09T04:26:55.436958+0000","last_active":"2023-10-09T04:27:35.180713+0000","last_peered":"2023-10-09T04:27:35.180713+0000","last_clean":"2023-10-09T04:27:35.180713+0000","last_became_active":"2023-10-09T04:26:55.436284+0000","last_became_peered":"2023-10-09T04:26:55.436284+0000","last_unstale":"2023-10-09T04:27:35.180713+0000","last_undegraded":"2023-10-09T04:27:35.180713+0000","last_fullsized":"2023-10-09T04:27:35.180713+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2418,"log_dups_size":0,"ondisk_log_size":2418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158466048,"num_objects":2418,"num_object_clones":0,"num_object_copies":7254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2418,"num_write_kb":154752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":157,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.440594+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:27:11.440594+0000","last_peered":"2023-10-09T04:27:11.440594+0000","last_clean":"2023-10-09T04:27:11.440594+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:27:11.440594+0000","last_undegraded":"2023-10-09T04:27:11.440594+0000","last_fullsized":"2023-10-09T04:27:11.440594+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202530+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:27:12.202530+0000","last_peered":"2023-10-09T04:27:12.202530+0000","last_clean":"2023-10-09T04:27:12.202530+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:27:12.202530+0000","last_undegraded":"2023-10-09T04:27:12.202530+0000","last_fullsized":"2023-10-09T04:27:12.202530+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930484+0000","last_change":"2023-10-09T04:11:27.247361+0000","last_active":"2023-10-09T04:27:11.930484+0000","last_peered":"2023-10-09T04:27:11.930484+0000","last_clean":"2023-10-09T04:27:11.930484+0000","last_became_active":"2023-10-09T04:11:27.247006+0000","last_became_peered":"2023-10-09T04:11:27.247006+0000","last_unstale":"2023-10-09T04:27:11.930484+0000","last_undegraded":"2023-10-09T04:27:11.930484+0000","last_fullsized":"2023-10-09T04:27:11.930484+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"110'2269","reported_seq":2282,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:33.815429+0000","last_change":"2023-10-09T04:26:55.433557+0000","last_active":"2023-10-09T04:27:33.815429+0000","last_peered":"2023-10-09T04:27:33.815429+0000","last_clean":"2023-10-09T04:27:33.815429+0000","last_became_active":"2023-10-09T04:26:55.433199+0000","last_became_peered":"2023-10-09T04:26:55.433199+0000","last_unstale":"2023-10-09T04:27:33.815429+0000","last_undegraded":"2023-10-09T04:27:33.815429+0000","last_fullsized":"2023-10-09T04:27:33.815429+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":2269,"log_dups_size":0,"ondisk_log_size":2269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:24.514947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148701184,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2269,"num_write_kb":145216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":49,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-10-09T04:27:13.530704+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:27:13.530704+0000","last_peered":"2023-10-09T04:27:13.530704+0000","last_clean":"2023-10-09T04:27:13.530704+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:27:13.530704+0000","last_undegraded":"2023-10-09T04:27:13.530704+0000","last_fullsized":"2023-10-09T04:27:13.530704+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202707+0000","last_change":"2023-10-09T04:24:00.811849+0000","last_active":"2023-10-09T04:27:12.202707+0000","last_peered":"2023-10-09T04:27:12.202707+0000","last_clean":"2023-10-09T04:27:12.202707+0000","last_became_active":"2023-10-09T04:24:00.811528+0000","last_became_peered":"2023-10-09T04:24:00.811528+0000","last_unstale":"2023-10-09T04:27:12.202707+0000","last_undegraded":"2023-10-09T04:27:12.202707+0000","last_fullsized":"2023-10-09T04:27:12.202707+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:25.981562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959390+0000","last_change":"2023-10-09T04:22:50.701405+0000","last_active":"2023-10-09T04:27:11.959390+0000","last_peered":"2023-10-09T04:27:11.959390+0000","last_clean":"2023-10-09T04:27:11.959390+0000","last_became_active":"2023-10-09T04:22:50.700948+0000","last_became_peered":"2023-10-09T04:22:50.700948+0000","last_unstale":"2023-10-09T04:27:11.959390+0000","last_undegraded":"2023-10-09T04:27:11.959390+0000","last_fullsized":"2023-10-09T04:27:11.959390+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.931303+0000","last_change":"2023-10-09T04:24:00.827442+0000","last_active":"2023-10-09T04:27:11.931303+0000","last_peered":"2023-10-09T04:27:11.931303+0000","last_clean":"2023-10-09T04:27:11.931303+0000","last_became_active":"2023-10-09T04:24:00.827036+0000","last_became_peered":"2023-10-09T04:24:00.827036+0000","last_unstale":"2023-10-09T04:27:11.931303+0000","last_undegraded":"2023-10-09T04:27:11.931303+0000","last_fullsized":"2023-10-09T04:27:11.931303+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.959726+0000","last_change":"2023-10-09T04:11:26.223020+0000","last_active":"2023-10-09T04:27:11.959726+0000","last_peered":"2023-10-09T04:27:11.959726+0000","last_clean":"2023-10-09T04:27:11.959726+0000","last_became_active":"2023-10-09T04:11:26.222714+0000","last_became_peered":"2023-10-09T04:11:26.222714+0000","last_unstale":"2023-10-09T04:27:11.959726+0000","last_undegraded":"2023-10-09T04:27:11.959726+0000","last_fullsized":"2023-10-09T04:27:11.959726+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930633+0000","last_change":"2023-10-09T04:24:00.827513+0000","last_active":"2023-10-09T04:27:11.930633+0000","last_peered":"2023-10-09T04:27:11.930633+0000","last_clean":"2023-10-09T04:27:11.930633+0000","last_became_active":"2023-10-09T04:24:00.827037+0000","last_became_peered":"2023-10-09T04:24:00.827037+0000","last_unstale":"2023-10-09T04:27:11.930633+0000","last_undegraded":"2023-10-09T04:27:11.930633+0000","last_fullsized":"2023-10-09T04:27:11.930633+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.438333+0000","last_change":"2023-10-09T04:24:00.819974+0000","last_active":"2023-10-09T04:27:11.438333+0000","last_peered":"2023-10-09T04:27:11.438333+0000","last_clean":"2023-10-09T04:27:11.438333+0000","last_became_active":"2023-10-09T04:24:00.819558+0000","last_became_peered":"2023-10-09T04:24:00.819558+0000","last_unstale":"2023-10-09T04:27:11.438333+0000","last_undegraded":"2023-10-09T04:27:11.438333+0000","last_fullsized":"2023-10-09T04:27:11.438333+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":99,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.437711+0000","last_change":"2023-10-09T04:24:00.815714+0000","last_active":"2023-10-09T04:27:11.437711+0000","last_peered":"2023-10-09T04:27:11.437711+0000","last_clean":"2023-10-09T04:27:11.437711+0000","last_became_active":"2023-10-09T04:24:00.815389+0000","last_became_peered":"2023-10-09T04:24:00.815389+0000","last_unstale":"2023-10-09T04:27:11.437711+0000","last_undegraded":"2023-10-09T04:27:11.437711+0000","last_fullsized":"2023-10-09T04:27:11.437711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:39.857138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":64,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:12.202796+0000","last_change":"2023-10-09T04:23:18.765967+0000","last_active":"2023-10-09T04:27:12.202796+0000","last_peered":"2023-10-09T04:27:12.202796+0000","last_clean":"2023-10-09T04:27:12.202796+0000","last_became_active":"2023-10-09T04:23:18.765587+0000","last_became_peered":"2023-10-09T04:23:18.765587+0000","last_unstale":"2023-10-09T04:27:12.202796+0000","last_undegraded":"2023-10-09T04:27:12.202796+0000","last_fullsized":"2023-10-09T04:27:12.202796+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:10.976153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T04:27:11.930203+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:27:11.930203+0000","last_peered":"2023-10-09T04:27:11.930203+0000","last_clean":"2023-10-09T04:27:11.930203+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:27:11.930203+0000","last_undegraded":"2023-10-09T04:27:11.930203+0000","last_fullsized":"2023-10-09T04:27:11.930203+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2398748672,"num_objects":36602,"num_object_clones":0,"num_object_copies":109806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36602,"num_write_kb":2342528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7309426688,"data_stored":7309426688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36602,"ondisk_log_size":36602,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430438,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078816,"kb_used_data":769304,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92666336,"statfs":{"total":95995035648,"available":94890328064,"internally_reserved":0,"allocated":787767296,"data_stored":787256568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.89600000000000002,"15min":0.89400000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.4020000000000001,"5min":3.4020000000000001,"15min":3.4020000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.96399999999999997,"15min":0.92900000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.145,"5min":2.3460000000000001,"15min":2.3460000000000001},"last":1.8100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:26:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.92300000000000004,"15min":0.92800000000000005},"min":{"1min":0.61899999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.3149999999999999,"5min":2.097,"15min":2.097},"last":1.3959999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.94999999999999996,"15min":0.93999999999999995},"min":{"1min":0.60799999999999998,"5min":0.503,"15min":0.503},"max":{"1min":1.2909999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.95399999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.95399999999999996,"15min":0.93400000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.8310000000000004,"5min":4.8310000000000004,"15min":4.8310000000000004},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.012,"15min":0.998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":3.0190000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":1.419}]},{"osd":3,"last update":"Mon Oct 9 04:27:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.018,"15min":1.024},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.256,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.093,"5min":1.0609999999999999,"15min":1.0449999999999999},"min":{"1min":0.83899999999999997,"5min":0.749,"15min":0.749},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.363}]},{"osd":4,"last update":"Mon Oct 9 04:27:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.93700000000000006,"15min":0.91900000000000004},"min":{"1min":0.625,"5min":0.55900000000000005,"15min":0.52900000000000003},"max":{"1min":1.2090000000000001,"5min":1.4550000000000001,"15min":2.0419999999999998},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.006,"5min":0.995,"15min":0.97299999999999998},"min":{"1min":0.73599999999999999,"5min":0.58499999999999996,"15min":0.495},"max":{"1min":1.3520000000000001,"5min":1.4399999999999999,"15min":2.1139999999999999},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0069999999999999,"15min":0.995},"min":{"1min":0.66700000000000004,"5min":0.51500000000000001,"15min":0.497},"max":{"1min":1.609,"5min":1.609,"15min":1.9510000000000001},"last":3.2639999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.0469999999999999,"15min":1.0289999999999999},"min":{"1min":0.81999999999999995,"5min":0.70299999999999996,"15min":0.45500000000000002},"max":{"1min":1.427,"5min":1.427,"15min":2.2349999999999999},"last":1.5800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.083,"15min":1.0660000000000001},"min":{"1min":0.88200000000000001,"5min":0.71999999999999997,"15min":0.46100000000000002},"max":{"1min":1.3660000000000001,"5min":1.5,"15min":2.3100000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.0700000000000001,"15min":1.0549999999999999},"min":{"1min":0.79100000000000004,"5min":0.54000000000000004,"15min":0.48199999999999998},"max":{"1min":1.446,"5min":1.4950000000000001,"15min":2.1909999999999998},"last":0.752}]}]},{"osd":6,"up_from":42,"seq":180388626671,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909208,"kb_used_data":607888,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92835944,"statfs":{"total":95995035648,"available":95064006656,"internally_reserved":0,"allocated":622477312,"data_stored":621974649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.879,"15min":0.88800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.89700000000000002,"15min":0.87},"min":{"1min":0.54500000000000004,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.079}]},{"osd":1,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.88500000000000001,"15min":0.89100000000000001},"min":{"1min":0.53900000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1759999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.91200000000000003,"15min":0.90200000000000002},"min":{"1min":0.65000000000000002,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.179,"5min":1.587,"15min":1.587},"last":0.51200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:27:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.93000000000000005,"15min":0.95699999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.143,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":2.8719999999999999},{"interface":"front","average":{"1min":0.876,"5min":0.91800000000000004,"15min":0.92200000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.1970000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.1220000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.91900000000000004,"15min":0.91700000000000004},"min":{"1min":0.73599999999999999,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.361,"5min":1.361,"15min":1.361},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.036,"5min":1.0409999999999999,"15min":1.04},"min":{"1min":0.73199999999999998,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.3959999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.72599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:27:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93700000000000006,"15min":0.91700000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.40899999999999997},"max":{"1min":1.321,"5min":1.367,"15min":1.663},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.99399999999999999,"15min":0.96399999999999997},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.443},"max":{"1min":1.411,"5min":1.411,"15min":1.548},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.0049999999999999,"15min":1.028},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.357,"5min":1.498,"15min":3.9180000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.04,"5min":1.0329999999999999,"15min":1.008},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.505},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.635},"last":0.56999999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:27:16 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.0429999999999999,"15min":1.0640000000000001},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.258,"5min":1.5569999999999999,"15min":1.9570000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.137,"15min":1.155},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.3959999999999999,"5min":1.6279999999999999,"15min":1.9990000000000001},"last":0.91600000000000004}]}]},{"osd":1,"up_from":93,"seq":399431958587,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757892,"kb_used_data":718012,"kb_used_omap":6,"kb_used_meta":39865,"kb_avail":92987260,"statfs":{"total":95995035648,"available":95218954240,"internally_reserved":0,"allocated":735244288,"data_stored":734702507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":40821772},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.68400000000000005,"15min":0.67700000000000005},"min":{"1min":0.34499999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.1080000000000001,"5min":1.1080000000000001,"15min":1.1080000000000001},"last":3.5710000000000002},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.71699999999999997,"15min":0.73699999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.97999999999999998,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":3.6680000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":0.78900000000000003,"15min":0.71499999999999997},"min":{"1min":0.42399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":1.3859999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.84799999999999998,"15min":0.80600000000000005},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.8109999999999999,"5min":8.8109999999999999,"15min":8.8109999999999999},"last":1.5800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83099999999999996,"15min":0.82699999999999996},"min":{"1min":0.51700000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.2170000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.2769999999999999},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.35299999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.93700000000000006,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":0.93400000000000005,"15min":0.92500000000000004},"min":{"1min":0.65400000000000003,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":1.2},{"interface":"front","average":{"1min":1.036,"5min":1.0209999999999999,"15min":1.038},"min":{"1min":0.71199999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.2390000000000001,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.361}]},{"osd":5,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.98099999999999998,"15min":1.0069999999999999},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.1799999999999999,"5min":1.819,"15min":1.819},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.04,"5min":0.98999999999999999,"15min":0.98799999999999999},"min":{"1min":0.73799999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.268,"5min":1.595,"15min":1.595},"last":1.0740000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.018,"15min":1.0269999999999999},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.2350000000000001,"5min":1.877,"15min":1.877},"last":0.498},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.014,"15min":1.022},"min":{"1min":0.69599999999999995,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.3779999999999999,"5min":1.724,"15min":1.724},"last":1.1759999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.012,"15min":1.0069999999999999},"min":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.331,"5min":1.653,"15min":1.653},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.1000000000000001,"15min":1.115},"min":{"1min":0.755,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.349,"5min":1.786,"15min":1.786},"last":1.329}]}]},{"osd":0,"up_from":88,"seq":377957122112,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097048,"kb_used_data":1065296,"kb_used_omap":9,"kb_used_meta":31734,"kb_avail":92648104,"statfs":{"total":95995035648,"available":94871658496,"internally_reserved":0,"allocated":1090863104,"data_stored":1090304062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":32496113},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.64000000000000001,"15min":0.64300000000000002},"min":{"1min":0.34300000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64500000000000002,"15min":0.66300000000000003},"min":{"1min":0.44600000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":1.1279999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.68600000000000005,"15min":0.67000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.706,"5min":3.706,"15min":3.706},"last":1.871},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.749,"15min":0.77300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.91200000000000003,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.311}]},{"osd":3,"last update":"Mon Oct 9 04:27:05 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.82299999999999995,"15min":0.82499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.161,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.71199999999999997,"15min":0.70999999999999996},"min":{"1min":0.39100000000000001,"5min":0.373,"15min":0.373},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.232}]},{"osd":4,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1,"15min":1.0640000000000001},"min":{"1min":0.68799999999999994,"5min":0.621,"15min":0.621},"max":{"1min":4.7480000000000002,"5min":4.7480000000000002,"15min":4.7480000000000002},"last":2.6259999999999999},{"interface":"front","average":{"1min":1.173,"5min":1.0649999999999999,"15min":1.0489999999999999},"min":{"1min":0.81299999999999994,"5min":0.63,"15min":0.63},"max":{"1min":3.6749999999999998,"5min":3.6749999999999998,"15min":3.6749999999999998},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.034,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":3.6339999999999999,"5min":3.6339999999999999,"15min":3.6339999999999999},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.238,"5min":1.0369999999999999,"15min":1.038},"min":{"1min":0.65500000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":3.343,"5min":3.343,"15min":3.343},"last":1.1819999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0049999999999999,"15min":1.0289999999999999},"min":{"1min":0.69299999999999995,"5min":0.622,"15min":0.622},"max":{"1min":1.875,"5min":1.875,"15min":1.875},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.071,"15min":1.032},"min":{"1min":0.69099999999999995,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":3.5800000000000001,"5min":3.5800000000000001,"15min":3.5800000000000001},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.1639999999999999,"15min":1.1459999999999999},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":10.528,"5min":10.528,"15min":10.528},"last":1.7110000000000001},{"interface":"front","average":{"1min":1.232,"5min":1.1020000000000001,"15min":1.137},"min":{"1min":0.69799999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":4.6950000000000003,"5min":4.6950000000000003,"15min":4.6950000000000003},"last":2.6640000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795061,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828852,"kb_used_data":788972,"kb_used_omap":11,"kb_used_meta":39860,"kb_avail":92916300,"statfs":{"total":95995035648,"available":95146291200,"internally_reserved":0,"allocated":807907328,"data_stored":807378953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":40816682},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.66200000000000003,"15min":0.68899999999999995},"min":{"1min":0.375,"5min":0.316,"15min":0.316},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.68999999999999995,"15min":0.66500000000000004},"min":{"1min":0.53600000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.6080000000000001,"5min":2.6080000000000001,"15min":2.6080000000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.70199999999999996,"15min":0.72899999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.97199999999999998,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.74199999999999999,"15min":0.72199999999999998},"min":{"1min":0.435,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.7210000000000001,"5min":2.7210000000000001,"15min":2.7210000000000001},"last":2.2749999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80600000000000005,"15min":0.81999999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.1279999999999999,"5min":2.2759999999999998,"15min":2.2759999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.85499999999999998,"15min":0.84899999999999998},"min":{"1min":0.438,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.1910000000000001,"5min":2.1899999999999999,"15min":2.1899999999999999},"last":2.21}]},{"osd":4,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.042,"15min":0.97899999999999998},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":6.2350000000000003,"5min":6.2350000000000003,"15min":6.2350000000000003},"last":2.2440000000000002},{"interface":"front","average":{"1min":1.0980000000000001,"5min":0.97199999999999998,"15min":0.96699999999999997},"min":{"1min":0.76300000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":1.3640000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0409999999999999,"15min":1.0309999999999999},"min":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.113,"5min":0.98499999999999999,"15min":0.96799999999999997},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":1.4350000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.0549999999999999,"15min":1.0369999999999999},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.569,"5min":4.569,"15min":4.569},"last":1.3959999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0129999999999999,"15min":1.0529999999999999},"min":{"1min":0.61799999999999999,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.1739999999999999,"5min":1.603,"15min":1.603},"last":0.96999999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.0669999999999999,"15min":1.123},"min":{"1min":0.65700000000000003,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.2410000000000001,"5min":1.825,"15min":1.825},"last":0.999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.099,"15min":1.0980000000000001},"min":{"1min":0.76200000000000001,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":3.1659999999999999,"5min":3.1659999999999999,"15min":3.1659999999999999},"last":2.298}]}]},{"osd":3,"up_from":103,"seq":442381631533,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":473516,"kb_used_data":433700,"kb_used_omap":5,"kb_used_meta":39802,"kb_avail":93271636,"statfs":{"total":95995035648,"available":95510155264,"internally_reserved":0,"allocated":444108800,"data_stored":443587561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":40757422},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.71599999999999997,"15min":0.76000000000000001},"min":{"1min":0.432,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.0289999999999999,"5min":1.615,"15min":1.615},"last":4.8250000000000002},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.70899999999999996,"15min":0.69799999999999995},"min":{"1min":0.50700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":1.173,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":4.8479999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68999999999999995,"15min":0.71199999999999997},"min":{"1min":0.40100000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":1.7090000000000001},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.77100000000000002,"15min":0.81899999999999995},"min":{"1min":0.36699999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.0469999999999999,"5min":2.145,"15min":2.145},"last":0.79300000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.78800000000000003,"15min":0.80900000000000005},"min":{"1min":0.39100000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.0960000000000001,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.7450000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.88,"15min":0.94599999999999995},"min":{"1min":0.47799999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.02,"5min":2.23,"15min":2.23},"last":1.5109999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.0209999999999999,"15min":1.073},"min":{"1min":0.64600000000000002,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.2330000000000001,"5min":2.0470000000000002,"15min":2.0470000000000002},"last":0.748},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.97099999999999997,"15min":0.99199999999999999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.27,"5min":1.345,"15min":1.345},"last":0.83399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0049999999999999,"15min":1.0369999999999999},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.25,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.032,"5min":1.04,"15min":1.0700000000000001},"min":{"1min":0.72399999999999998,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.218,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":1.875}]},{"osd":6,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.006,"15min":1.0409999999999999},"min":{"1min":0.69399999999999995,"5min":0.505,"15min":0.505},"max":{"1min":1.3600000000000001,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.008,"5min":1.0569999999999999,"15min":1.0920000000000001},"min":{"1min":0.77600000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":1.635}]},{"osd":7,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.105,"15min":1.131},"min":{"1min":0.77700000000000002,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.413,"5min":1.498,"15min":1.498},"last":1.677},{"interface":"front","average":{"1min":1.054,"5min":1.155,"15min":1.2150000000000001},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.3220000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":0.77400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019135,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1550936,"kb_used_data":1241424,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92194216,"statfs":{"total":95995035648,"available":94406877184,"internally_reserved":0,"allocated":1271218176,"data_stored":1270715026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":1.3859999999999999,"15min":1.0369999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":48.450000000000003,"5min":48.450000000000003,"15min":48.450000000000003},"last":48.450000000000003},{"interface":"front","average":{"1min":3.7469999999999999,"5min":1.452,"15min":1.0900000000000001},"min":{"1min":0.66700000000000004,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":49.283000000000001,"5min":49.283000000000001,"15min":49.283000000000001},"last":49.283000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.88200000000000001,"15min":0.90200000000000002},"min":{"1min":0.628,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1890000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.88900000000000001,"15min":0.89000000000000001},"min":{"1min":0.55700000000000005,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.0880000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.60099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:23 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.95299999999999996,"15min":0.93300000000000005},"min":{"1min":0.68899999999999995,"5min":0.502,"15min":0.502},"max":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":48.323},{"interface":"front","average":{"1min":0.877,"5min":0.94299999999999995,"15min":0.95399999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.1559999999999999,"5min":1.272,"15min":1.272},"last":48.289000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.024,"15min":1.032},"min":{"1min":0.70599999999999996,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.2509999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.0649999999999999,"15min":1.079},"min":{"1min":0.77400000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.319,"5min":1.363,"15min":1.363},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:26:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.96499999999999997,"15min":0.94399999999999995},"min":{"1min":0.66700000000000004,"5min":0.59999999999999998,"15min":0.441},"max":{"1min":1.371,"5min":1.5269999999999999,"15min":1.5509999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.95299999999999996,"15min":0.90900000000000003},"min":{"1min":0.66100000000000003,"5min":0.498,"15min":0.498},"max":{"1min":1.258,"5min":1.877,"15min":1.877},"last":0.69599999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.002,"15min":0.97799999999999998},"min":{"1min":0.58999999999999997,"5min":0.56100000000000005,"15min":0.47099999999999997},"max":{"1min":1.796,"5min":1.796,"15min":1.796},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0660000000000001,"15min":1.0389999999999999},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.393,"5min":1.583,"15min":1.796},"last":0.64500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.155,"15min":1.125},"min":{"1min":0.84199999999999997,"5min":0.82599999999999996,"15min":0.65900000000000003},"max":{"1min":1.46,"5min":1.653,"15min":1.8600000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1379999999999999,"15min":1.115},"min":{"1min":0.86399999999999999,"5min":0.76500000000000001,"15min":0.57699999999999996},"max":{"1min":1.345,"5min":1.512,"15min":1.923},"last":0.68300000000000005}]}]},{"osd":5,"up_from":36,"seq":154618822902,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1697864,"kb_used_data":1396544,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92047288,"statfs":{"total":95995035648,"available":94256422912,"internally_reserved":0,"allocated":1430061056,"data_stored":1429569738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8919999999999999,"5min":1.474,"15min":1.073},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":53.731000000000002,"5min":53.731000000000002,"15min":53.731000000000002},"last":53.731000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":1.5429999999999999,"15min":1.0620000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":53.222999999999999,"5min":53.222999999999999,"15min":53.222999999999999},"last":53.222999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:26:56 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.96299999999999997,"15min":0.99099999999999999},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2589999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.96899999999999997,"15min":0.95599999999999996},"min":{"1min":0.60299999999999998,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.204,"5min":1.645,"15min":1.645},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:27:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.94699999999999995,"15min":0.94099999999999995},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":53.692999999999998},{"interface":"front","average":{"1min":1.052,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":2.5390000000000001,"5min":2.5390000000000001,"15min":2.5390000000000001},"last":53.616999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.996,"15min":0.98999999999999999},"min":{"1min":0.73599999999999999,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.2050000000000001,"5min":1.502,"15min":1.502},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.998,"5min":1.046,"15min":1.048},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.254,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.61199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:26:42 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.94499999999999995,"15min":0.94499999999999995},"min":{"1min":0.65200000000000002,"5min":0.51600000000000001,"15min":0.47099999999999997},"max":{"1min":1.1319999999999999,"5min":1.383,"15min":2.165},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.022,"15min":0.999},"min":{"1min":0.73999999999999999,"5min":0.59399999999999997,"15min":0.47599999999999998},"max":{"1min":1.343,"5min":1.4319999999999999,"15min":3.7989999999999999},"last":0.876}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.998,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.9849999999999999,"5min":3.9849999999999999,"15min":3.9849999999999999},"last":0.752},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0740000000000001,"15min":1.0660000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.847,"5min":1.847,"15min":2.5840000000000001},"last":0.90600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.151,"15min":1.119},"min":{"1min":0.78200000000000003,"5min":0.67600000000000005,"15min":0.42599999999999999},"max":{"1min":3.992,"5min":3.992,"15min":3.992},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.083,"15min":1.0920000000000001},"min":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.47799999999999998},"max":{"1min":1.417,"5min":1.4850000000000001,"15min":2.2829999999999999},"last":0.85699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1107820544,"data_stored":1107820544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":756678656,"data_stored":756678656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":839122944,"data_stored":839122944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":452001792,"data_stored":452001792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1275789312,"data_stored":1275789312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1439170560,"data_stored":1439170560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":634519552,"data_stored":634519552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":804323328,"data_stored":804323328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:27:40.639 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:27:40.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T04:27:40.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:40.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:40 smithi153 bash[59565]: cluster 2023-10-09T04:27:39.359902+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:27:40.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:40.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:40.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:40.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:40.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:40.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 47364 47348 118.352 92.9375 0.0092452 0.00844347 2023-10-09T04:27:40.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:41.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:41.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:41.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:41.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:41.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:41.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:41.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:41.534 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:41.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:41.673 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:41.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:41.808 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:41.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:41.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 48836 48820 117.338 92 0.0113099 0.00851669 2023-10-09T04:27:41.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:41.980 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:41.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:42.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:42.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:42.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:42.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:42.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:42.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:42.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:42.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:27:42.580 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:42 smithi082 bash[77063]: cluster 2023-10-09T04:27:41.360579+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:27:42.582 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:42 smithi082 bash[79733]: cluster 2023-10-09T04:27:41.360579+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:27:42.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:27:42.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:42.730 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:42.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.846 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:42 smithi153 bash[59565]: cluster 2023-10-09T04:27:41.360579+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 2.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:27:42.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:42.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:42.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 50212 50196 116.177 86 0.0185814 0.00860136 2023-10-09T04:27:42.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:43.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:43.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:43.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:43.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.263 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:43.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:43.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:43.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:43.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:43.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:43.766 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:43.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:43.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:43.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:43.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:43.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:43.947 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 16 51643 51627 115.222 89.4375 0.0105412 0.00867336 2023-10-09T04:27:44.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:44.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:44.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:44.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:44.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:44.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:44.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:44.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:44.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:44.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:44 smithi082 bash[77063]: cluster 2023-10-09T04:27:43.361185+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T04:27:44.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:44 smithi082 bash[77063]: audit 2023-10-09T04:27:44.256129+0000 mon.a (mon.0) 566 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:44 smithi082 bash[79733]: cluster 2023-10-09T04:27:43.361185+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T04:27:44.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:44 smithi082 bash[79733]: audit 2023-10-09T04:27:44.256129+0000 mon.a (mon.0) 566 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.611 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:44.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:44.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:44.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:44.789 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:44.857 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:44.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:44.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:44.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:44 smithi153 bash[59565]: cluster 2023-10-09T04:27:43.361185+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T04:27:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:44 smithi153 bash[59565]: audit 2023-10-09T04:27:44.256129+0000 mon.a (mon.0) 566 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 53007 52991 114.188 85.25 0.0082725 0.00875263 2023-10-09T04:27:44.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:45.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:45.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:45.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:45.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:45.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:45.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:45.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:45.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:45.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:45.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:45.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:45.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:45.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:45.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:45.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:45.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:45.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 54273 54257 113.019 79.125 0.0267744 0.0088428 2023-10-09T04:27:45.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:45.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:46.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:46.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:46.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:46.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:46.218 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:46.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:46.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:46.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:46.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:46.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:46.593 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:46 smithi082 bash[77063]: cluster 2023-10-09T04:27:45.361890+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:27:46.593 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:46 smithi082 bash[79733]: cluster 2023-10-09T04:27:45.361890+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:27:46.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:46.722 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:46.729 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:46 smithi153 bash[59565]: cluster 2023-10-09T04:27:45.361890+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:27:46.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:46.822 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:46.826 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:46.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:46.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:46.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:46.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 55627 55611 112.103 84.625 0.010873 0.00891449 2023-10-09T04:27:46.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:47.011 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:47.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:47.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:47.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:47.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:47.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:47.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:47.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:47.528 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:47.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:47.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:47.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:47.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:47.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:47.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:47.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:47.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 56727 56711 110.748 68.75 0.0111915 0.0090228 2023-10-09T04:27:48.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:48.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:48.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:48.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:48.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:48.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:48.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:48.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:48.571 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:48.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:48.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:48.729 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:48.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:48.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:48.842 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:48 smithi082 bash[77063]: cluster 2023-10-09T04:27:47.362418+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:27:48.842 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:48 smithi082 bash[79733]: cluster 2023-10-09T04:27:47.362418+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:27:48.862 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:48.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:48.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:48 smithi153 bash[59565]: cluster 2023-10-09T04:27:47.362418+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v339: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:27:48.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 57850 57834 109.519 70.1875 0.0113319 0.00912448 2023-10-09T04:27:48.966 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:48.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:49.101 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:49.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:49.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:49.357 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:49.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:49.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:49.653 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:49.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:49.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:49.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:49.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:49.894 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:49] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-09T04:27:49.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:49.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:49.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 59028 59012 108.463 73.625 0.0118672 0.00921394 2023-10-09T04:27:50.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:50.041 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:50.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:50.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:50.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:50.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:50.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:50.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:50.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:50.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.572 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:50.581 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:50 smithi082 bash[79733]: cluster 2023-10-09T04:27:49.363056+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:27:50.584 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:50 smithi082 bash[77063]: cluster 2023-10-09T04:27:49.363056+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:27:50.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:50.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:50.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:50.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:50 smithi153 bash[59565]: cluster 2023-10-09T04:27:49.363056+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:27:50.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 60236 60220 107.521 75.5 0.00700709 0.00929469 2023-10-09T04:27:50.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:50.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:51.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:51.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:51.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.227 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:51.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:51.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:51.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:51.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:51.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:51.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:51.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:51.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:51.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:51.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 61318 61302 106.413 67.625 0.0127249 0.00939162 2023-10-09T04:27:52.013 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:52.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:52.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:52.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:52.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:52.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:52.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:52.545 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:52 smithi082 bash[77063]: cluster 2023-10-09T04:27:51.363594+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:27:52.546 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:52 smithi082 bash[79733]: cluster 2023-10-09T04:27:51.363594+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:27:52.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:52.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:52.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:52.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:52.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:52.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:52 smithi153 bash[59565]: cluster 2023-10-09T04:27:51.363594+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:27:52.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 62486 62470 105.509 73 0.00933316 0.00947203 2023-10-09T04:27:52.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:52.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:53.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:53.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:53.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:53.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:53.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:53.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:53.367 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:53.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:53.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:53.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:53.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:53.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:53.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:53.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:53.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:53.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:53.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 63737 63721 104.79 78.1875 0.0124958 0.00953748 2023-10-09T04:27:53.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:54.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:54.137 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:54.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:54.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:54.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:54.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:54.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:54.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:54.572 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:54 smithi153 bash[59565]: cluster 2023-10-09T04:27:53.364015+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:27:54.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:54.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:54.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:54 smithi082 bash[77063]: cluster 2023-10-09T04:27:53.364015+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:27:54.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:54 smithi082 bash[79733]: cluster 2023-10-09T04:27:53.364015+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:27:54.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:54.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:54.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:54.938 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:54.948 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 64951 64935 104.049 75.875 0.00781632 0.00960535 2023-10-09T04:27:54.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:54.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:54.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:55.066 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:55.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:55.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:55.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:55.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:55.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:55.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:55.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:55.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:55.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:55.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:55.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:55.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:55.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:55.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:55.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:55.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:27:55.948408+0000 min lat: 0.00135884 max lat: 0.18495 avg lat: 0.00967598 2023-10-09T04:27:55.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:27:55.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 66132 66116 103.293 73.8125 0.0116194 0.00967598 2023-10-09T04:27:56.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:56.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:56.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:56.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:56.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:56.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:56.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:56.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:56.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:56.703 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:56 smithi153 bash[59565]: cluster 2023-10-09T04:27:55.364817+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T04:27:56.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:56.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:56 smithi082 bash[77063]: cluster 2023-10-09T04:27:55.364817+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T04:27:56.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:56 smithi082 bash[79733]: cluster 2023-10-09T04:27:55.364817+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T04:27:56.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:56.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 67312 67296 102.572 73.75 0.00656958 0.00974312 2023-10-09T04:27:56.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:56.980 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:56.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:57.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:57.108 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:57.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:57.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:57.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:57.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:57.382 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:57.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.528 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:57.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:57.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:57.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:57.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:57.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:57.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:57.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:57.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:57.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 68227 68211 101.491 57.1875 0.00940173 0.00983184 2023-10-09T04:27:58.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:58.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:58.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:58.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:58.275 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:58.293 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:58.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:58.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:58.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:58.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:58.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:58.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:58 smithi153 bash[59565]: cluster 2023-10-09T04:27:57.365436+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:27:58.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:58 smithi082 bash[77063]: cluster 2023-10-09T04:27:57.365436+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:27:58.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:58 smithi082 bash[79733]: cluster 2023-10-09T04:27:57.365436+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:27:58.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:58.838 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:58.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:58.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:58.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 69306 69290 100.699 67.4375 0.00850356 0.00992536 2023-10-09T04:27:58.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:58.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:59.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:59.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:59.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:59.240 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:59.293 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:59.293 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:59.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:59.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:59.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:59.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:59.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:59.746 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:27:59 smithi082 bash[77063]: audit 2023-10-09T04:27:59.256687+0000 mon.a (mon.0) 567 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:59.747 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:27:59 smithi082 bash[79733]: audit 2023-10-09T04:27:59.256687+0000 mon.a (mon.0) 567 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:59.747 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:27:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:27:59] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-10-09T04:27:59.751 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:27:59 smithi153 bash[59565]: audit 2023-10-09T04:27:59.256687+0000 mon.a (mon.0) 567 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:59.783 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:59.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:59.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:59.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:27:59.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:59.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 70379 70363 99.9345 67.0625 0.00898778 0.010001 2023-10-09T04:28:00.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:00.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:00.180 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:28:00.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:00.212 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:28:00.256 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:28:00.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.296 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:28:00.316 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:00.329 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:28:00.373 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:28:00.413 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:28:00.421 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.445 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.447 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.448 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.448 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.448 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:00.523 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:28:00.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:00.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:00 smithi153 bash[59565]: cluster 2023-10-09T04:27:59.365999+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:00 smithi153 bash[59565]: audit 2023-10-09T04:28:00.132780+0000 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.82:0/461482691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:00 smithi153 bash[59565]: audit 2023-10-09T04:28:00.438951+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:00.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:00.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:00.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:00.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:00.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 71417 71401 99.1553 64.875 0.00344026 0.0100755 2023-10-09T04:28:00.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[77063]: cluster 2023-10-09T04:27:59.365999+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[77063]: audit 2023-10-09T04:28:00.132780+0000 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.82:0/461482691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[77063]: audit 2023-10-09T04:28:00.438951+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:00.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[79733]: cluster 2023-10-09T04:27:59.365999+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:00.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[79733]: audit 2023-10-09T04:28:00.132780+0000 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.82:0/461482691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:00.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:00 smithi082 bash[79733]: audit 2023-10-09T04:28:00.438951+0000 mon.a (mon.0) 569 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:00.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:00.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:01.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.126 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T04:28:01.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:01.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:01.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:01.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:01.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:01.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:01.498 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: cluster 2023-10-09T04:28:00.493542+0000 mon.a (mon.0) 570 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: cluster 2023-10-09T04:28:00.493576+0000 mon.a (mon.0) 571 : cluster 1 Cluster is now healthy 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: cluster 2023-10-09T04:28:00.509582+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: audit 2023-10-09T04:28:00.514830+0000 mon.a (mon.0) 572 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: cluster 2023-10-09T04:28:00.515124+0000 mon.a (mon.0) 573 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: cluster 2023-10-09T04:28:00.527198+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: audit 2023-10-09T04:28:01.121593+0000 mon.c (mon.2) 42 : audit 1 from='client.? 172.21.15.82:0/2217044728' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:01 smithi153 bash[59565]: audit 2023-10-09T04:28:01.122283+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:01.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:01.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:01.654 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:01.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: cluster 2023-10-09T04:28:00.493542+0000 mon.a (mon.0) 570 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:01.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: cluster 2023-10-09T04:28:00.493576+0000 mon.a (mon.0) 571 : cluster 1 Cluster is now healthy 2023-10-09T04:28:01.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: cluster 2023-10-09T04:28:00.509582+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: audit 2023-10-09T04:28:00.514830+0000 mon.a (mon.0) 572 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: cluster 2023-10-09T04:28:00.515124+0000 mon.a (mon.0) 573 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: cluster 2023-10-09T04:28:00.527198+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: audit 2023-10-09T04:28:01.121593+0000 mon.c (mon.2) 42 : audit 1 from='client.? 172.21.15.82:0/2217044728' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[77063]: audit 2023-10-09T04:28:01.122283+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: cluster 2023-10-09T04:28:00.493542+0000 mon.a (mon.0) 570 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: cluster 2023-10-09T04:28:00.493576+0000 mon.a (mon.0) 571 : cluster 1 Cluster is now healthy 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: cluster 2023-10-09T04:28:00.509582+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: audit 2023-10-09T04:28:00.514830+0000 mon.a (mon.0) 572 : audit 1 from='client.? 172.21.15.82:0/3975632216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: cluster 2023-10-09T04:28:00.515124+0000 mon.a (mon.0) 573 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:28:01.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: cluster 2023-10-09T04:28:00.527198+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T04:28:01.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: audit 2023-10-09T04:28:01.121593+0000 mon.c (mon.2) 42 : audit 1 from='client.? 172.21.15.82:0/2217044728' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:01 smithi082 bash[79733]: audit 2023-10-09T04:28:01.122283+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:28:01.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:01.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:28:01.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:01.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:01.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 72405 72389 98.3421 61.75 0.00879712 0.0101637 2023-10-09T04:28:02.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:02.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:02.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:02.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:02.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:02.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:02.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:02.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:02 smithi082 bash[79733]: cluster 2023-10-09T04:28:01.366442+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:28:02.614 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:02 smithi082 bash[77063]: cluster 2023-10-09T04:28:01.366442+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:28:02.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:02.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:02 smithi153 bash[59565]: cluster 2023-10-09T04:28:01.366442+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:28:02.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:02.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:02.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:02.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:02.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:02.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 16 73559 73543 97.7842 72.125 0.0125497 0.0102216 2023-10-09T04:28:02.998 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:03.019 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:03.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:03.154 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:03.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:03.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:03.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:03.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:03.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:03.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:03.527 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:03.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:03.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:03.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:03.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T04:28:03.829 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:03.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:03.949 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 74662 74646 97.1831 68.9375 0.0044872 0.0102844 2023-10-09T04:28:03.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:03.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:04.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:04.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:04.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:04.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:04.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:04.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:04.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:04.516 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:04 smithi153 bash[59565]: cluster 2023-10-09T04:28:03.367062+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:04.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.615 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:04 smithi082 bash[79733]: cluster 2023-10-09T04:28:03.367062+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:04.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:04 smithi082 bash[77063]: cluster 2023-10-09T04:28:03.367062+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:04.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:04.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:04.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:04.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:04.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:04.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 75763 75747 96.604 68.8125 0.0126327 0.0103464 2023-10-09T04:28:05.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:05.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:05.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:05.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:05.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:05.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:05.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:05.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:05.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:05.586 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:05.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:28:05.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:05.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:05.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:05.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:05.881 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:05.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:05.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:05.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 76829 76813 96.0043 66.625 0.0177425 0.0104104 2023-10-09T04:28:06.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:06.060 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:06.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:06.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:06.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:06.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:06.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:06.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:06.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:06.571 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:06.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:06.672 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:06.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:06.713 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:06 smithi153 bash[59565]: cluster 2023-10-09T04:28:05.367889+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:28:06.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:06.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:06.853 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:06 smithi082 bash[77063]: cluster 2023-10-09T04:28:05.367889+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:28:06.853 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:06 smithi082 bash[79733]: cluster 2023-10-09T04:28:05.367889+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:28:06.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:06.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:06.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 77881 77865 95.4109 65.75 0.00599169 0.0104757 2023-10-09T04:28:06.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:07.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:07.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:07.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:07.373 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:07.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:07.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:07.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:07.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:07.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:07.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:07.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 78924 78908 94.8296 65.1875 0.0054165 0.0105388 2023-10-09T04:28:07.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:07.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:08.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:08.087 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:08.191 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:08.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:08.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:08.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:08.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:08.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:08.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:08.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:08.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:08 smithi153 bash[59565]: cluster 2023-10-09T04:28:07.368401+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:08.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:08.730 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:08 smithi082 bash[77063]: cluster 2023-10-09T04:28:07.368401+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:08.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:08 smithi082 bash[79733]: cluster 2023-10-09T04:28:07.368401+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:08.750 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:08.803 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:28:08.803 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:28:08.803 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:28:08.804 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:28:08.804 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:28:08.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:28:08.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:08.887 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:08.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:08.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 79969 79953 94.2726 65.3125 0.0475802 0.0106018 2023-10-09T04:28:08.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:09.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:09.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:09.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:09.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:09.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:09.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:09.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:09.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:09.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:09.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:09.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:09.682 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:09.786 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:09.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:09.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:09.924 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:09] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.33.4" 2023-10-09T04:28:09.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:09.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:09.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 80963 80947 93.6772 62.125 0.0111253 0.0106689 2023-10-09T04:28:10.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:10.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:10.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:10.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:10.259 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:10.265 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:10.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:10.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:10.456 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:10.488 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:10.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:10.601 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:10 smithi082 bash[77063]: cluster 2023-10-09T04:28:09.369057+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:10.601 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:10 smithi082 bash[79733]: cluster 2023-10-09T04:28:09.369057+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:10.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:10.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:10 smithi153 bash[59565]: cluster 2023-10-09T04:28:09.369057+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:10.718 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:10.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:10.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:10.876 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:10.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:10.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 82020 82004 93.175 66.0625 0.0170409 0.010727 2023-10-09T04:28:10.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:11.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:11.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:11.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:11.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:11.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:11.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:11.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:11.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:11.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:11.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:11.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:11.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:11.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:11.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:11.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 83043 83027 92.6528 63.9375 0.0123624 0.010788 2023-10-09T04:28:12.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:12.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:12.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:12.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:12.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.723 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:12 smithi153 bash[59565]: cluster 2023-10-09T04:28:11.369742+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T04:28:12.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:12.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:12.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:12.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 83984 83968 92.0591 58.8125 0.00789042 0.0108575 2023-10-09T04:28:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:12 smithi082 bash[77063]: cluster 2023-10-09T04:28:11.369742+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T04:28:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:12 smithi082 bash[79733]: cluster 2023-10-09T04:28:11.369742+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T04:28:12.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:13.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:13.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:13.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:13.269 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:13.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:13.381 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:28:13.382 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:28:13.382 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:28:13.382 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:28:13.382 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:28:13.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:13.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:13.531 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:13.566 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:13.629 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:13.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:13.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:13.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:13.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:13.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:13.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 84965 84949 91.5288 61.3125 0.00740956 0.01092 2023-10-09T04:28:13.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:13.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.070 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:14.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:14.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.499 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:14.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:14.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[77063]: cluster 2023-10-09T04:28:13.370248+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v353: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[77063]: audit 2023-10-09T04:28:14.261308+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[77063]: audit 2023-10-09T04:28:14.262520+0000 mon.a (mon.0) 576 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[79733]: cluster 2023-10-09T04:28:13.370248+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v353: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[79733]: audit 2023-10-09T04:28:14.261308+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:14.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:14 smithi082 bash[79733]: audit 2023-10-09T04:28:14.262520+0000 mon.a (mon.0) 576 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.776 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:14.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.918 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:14.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:14 smithi153 bash[59565]: cluster 2023-10-09T04:28:13.370248+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v353: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:14 smithi153 bash[59565]: audit 2023-10-09T04:28:14.261308+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:14 smithi153 bash[59565]: audit 2023-10-09T04:28:14.262520+0000 mon.a (mon.0) 576 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.950 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 85918 85902 90.987 59.5625 0.00463444 0.010985 2023-10-09T04:28:15.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:15.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:15.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:15.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:15.291 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:15.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:15.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:15.448 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:15.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:15.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:15.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:15.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:15.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:15.831 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:15.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:15.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:15.951 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:28:15.950299+0000 min lat: 0.00135884 max lat: 0.18495 avg lat: 0.0110406 2023-10-09T04:28:15.951 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:28:15.951 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 86935 86919 90.5298 63.5625 0.0399192 0.0110406 2023-10-09T04:28:15.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:16.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:16.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:28:16.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:16.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:16.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:16.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:16.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:16.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:16.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:16.544 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:16 smithi153 bash[59565]: cluster 2023-10-09T04:28:15.370923+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:16.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:16.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:16.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:16.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:16.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:16 smithi082 bash[79733]: cluster 2023-10-09T04:28:15.370923+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:16.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:16 smithi082 bash[77063]: cluster 2023-10-09T04:28:15.370923+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:28:16.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:16.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:16.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:16.951 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 87829 87813 89.9604 55.875 0.0126682 0.01111 2023-10-09T04:28:16.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:16.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:17.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:17.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:17.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:17.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:17.343 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:17.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:17.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:17.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:17.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.658 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:17.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:17.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:17.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:17.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:17.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:17.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:17.952 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 88815 88799 89.5033 61.625 0.0276803 0.0111665 2023-10-09T04:28:18.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:18.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:18.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:18.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:18.284 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:28:18.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:18.319 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:28:18.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:18.372 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:28:18.422 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:28:18.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:18.461 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:28:18.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:18.503 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:28:18.545 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:28:18.575 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:18.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:18 smithi153 bash[59565]: cluster 2023-10-09T04:28:17.371449+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:18.577 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:18 smithi153 bash[59565]: audit 2023-10-09T04:28:18.239696+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.82:0/218119259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:18.582 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:28:18.585 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.585 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.586 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:28:18.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:18.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:18.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:18.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:18.854 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:18 smithi082 bash[77063]: cluster 2023-10-09T04:28:17.371449+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:18.854 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:18 smithi082 bash[77063]: audit 2023-10-09T04:28:18.239696+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.82:0/218119259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:18.854 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:18 smithi082 bash[79733]: cluster 2023-10-09T04:28:17.371449+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:18.855 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:18 smithi082 bash[79733]: audit 2023-10-09T04:28:18.239696+0000 mon.a (mon.0) 577 : audit 0 from='client.? 172.21.15.82:0/218119259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:18.888 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:18.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:18.952 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 89675 89659 88.9357 53.75 0.0263452 0.0112383 2023-10-09T04:28:18.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:19.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:19.130 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:19.160 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:19.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:19.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:19.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:19.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:19.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:19.571 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:28:19.658 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.691 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:19.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:19.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:19 smithi082 bash[77063]: audit 2023-10-09T04:28:18.656136+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.82:0/1848034898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:19 smithi082 bash[77063]: audit 2023-10-09T04:28:18.656657+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:19 smithi082 bash[79733]: audit 2023-10-09T04:28:18.656136+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.82:0/1848034898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:19 smithi082 bash[79733]: audit 2023-10-09T04:28:18.656657+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.811 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:19] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.33.4" 2023-10-09T04:28:19.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:19.911 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:28:19.911 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:28:19.911 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:28:19.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:19 smithi153 bash[59565]: audit 2023-10-09T04:28:18.656136+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.82:0/1848034898' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:19 smithi153 bash[59565]: audit 2023-10-09T04:28:18.656657+0000 mon.a (mon.0) 578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:19.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:19.952 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 90641 90625 88.4893 60.375 0.00461456 0.0112945 2023-10-09T04:28:19.957 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:19.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:20.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:20.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:20.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:20.325 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:20.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:20.598 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:20.611 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: cluster 2023-10-09T04:28:19.371987+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v356: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T04:28:20.612 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: cluster 2023-10-09T04:28:19.547435+0000 mon.a (mon.0) 579 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:20.612 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: cluster 2023-10-09T04:28:19.560268+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:20.612 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: audit 2023-10-09T04:28:19.564697+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:20.612 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: cluster 2023-10-09T04:28:19.564814+0000 mon.a (mon.0) 581 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:28:20.612 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:20 smithi153 bash[59565]: cluster 2023-10-09T04:28:19.574077+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:20.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:20.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:20.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:20.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:20.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:20.912 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:20.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:20.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:20.952 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 91559 91543 88.0103 57.375 0.0154669 0.011357 2023-10-09T04:28:20.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: cluster 2023-10-09T04:28:19.371987+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v356: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T04:28:20.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: cluster 2023-10-09T04:28:19.547435+0000 mon.a (mon.0) 579 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: cluster 2023-10-09T04:28:19.560268+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: audit 2023-10-09T04:28:19.564697+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: cluster 2023-10-09T04:28:19.564814+0000 mon.a (mon.0) 581 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[77063]: cluster 2023-10-09T04:28:19.574077+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: cluster 2023-10-09T04:28:19.371987+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v356: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: cluster 2023-10-09T04:28:19.547435+0000 mon.a (mon.0) 579 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: cluster 2023-10-09T04:28:19.560268+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: audit 2023-10-09T04:28:19.564697+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:20.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: cluster 2023-10-09T04:28:19.564814+0000 mon.a (mon.0) 581 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:28:20.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:20 smithi082 bash[79733]: cluster 2023-10-09T04:28:19.574077+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T04:28:21.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:21.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:21.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.298 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:21.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:21.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:21.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:21.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:21.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:21.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:28:21.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:21.806 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:21.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:21.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.952 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 92498 92482 87.566 58.6875 0.0272408 0.0114139 2023-10-09T04:28:21.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:21.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:22.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:22.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:22.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:22.240 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:22.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:22.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:22.482 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:22.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:28:22.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:22.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:22.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:22.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:22 smithi082 bash[77063]: cluster 2023-10-09T04:28:21.372596+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T04:28:22.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:22 smithi082 bash[79733]: cluster 2023-10-09T04:28:21.372596+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T04:28:22.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:22.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:22 smithi153 bash[59565]: cluster 2023-10-09T04:28:21.372596+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v358: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T04:28:22.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:22.954 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 93437 93421 87.1335 58.6875 0.00950195 0.0114705 2023-10-09T04:28:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:22.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:23.035 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:23.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:23.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:23.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:23.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:23.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:23.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:23.379 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:23.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:23.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:23.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:23.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:23.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:23.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:23.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:23.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:23.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:23.953 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 94394 94378 86.7317 59.8125 0.00705262 0.0115247 2023-10-09T04:28:24.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:24.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:24.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:24.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:24.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:24.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:24.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:24.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:24.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:24.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:24.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:24.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:24.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:24.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:24.831 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:24 smithi153 bash[59565]: cluster 2023-10-09T04:28:23.373147+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v359: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T04:28:24.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:24.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:24 smithi082 bash[77063]: cluster 2023-10-09T04:28:23.373147+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v359: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T04:28:24.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:24 smithi082 bash[79733]: cluster 2023-10-09T04:28:23.373147+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v359: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T04:28:24.954 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 16 95334 95318 86.3261 58.75 0.0150273 0.0115786 2023-10-09T04:28:24.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:24.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:24.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:25.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:25.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:25.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:25.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:25.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:25.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:25.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:25.503 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:25.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:25.647 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:25.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:25.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:25.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:25.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:25.917 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:28:25.918 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:28:25.918 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:28:25.918 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T04:28:25.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T04:28:25.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:25.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:25.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 96269 96253 85.9233 58.4375 0.0128501 0.0116329 2023-10-09T04:28:26.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:26.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:26.182 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:26.193 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:25 smithi153 systemd[1]: Stopping Ceph osd.5 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:28:26.193 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:26 smithi153 bash[35557]: debug 2023-10-09T04:28:26.101+0000 7fd9caa5d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:28:26.193 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:26 smithi153 bash[35557]: debug 2023-10-09T04:28:26.101+0000 7fd9caa5d700 -1 osd.5 113 *** Got signal Terminated *** 2023-10-09T04:28:26.193 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:26 smithi153 bash[35557]: debug 2023-10-09T04:28:26.101+0000 7fd9caa5d700 -1 osd.5 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:28:26.212 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:26.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:26.345 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:26.399 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:26.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:26.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:26.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:26.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:26.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:26.735 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:26 smithi153 bash[59565]: cluster 2023-10-09T04:28:25.373957+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v360: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:28:26.735 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:26 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.111066+0000 mon.a (mon.0) 582 : cluster 1 osd.5 marked itself down and dead 2023-10-09T04:28:26.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:26.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:26.858 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:26 smithi082 bash[77063]: cluster 2023-10-09T04:28:25.373957+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v360: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:28:26.858 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:26 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.111066+0000 mon.a (mon.0) 582 : cluster 1 osd.5 marked itself down and dead 2023-10-09T04:28:26.858 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:26 smithi082 bash[79733]: cluster 2023-10-09T04:28:25.373957+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v360: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:28:26.859 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:26 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.111066+0000 mon.a (mon.0) 582 : cluster 1 osd.5 marked itself down and dead 2023-10-09T04:28:26.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:26.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:26.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 16 96865 96849 85.2378 37.25 0.0149824 0.0116676 2023-10-09T04:28:26.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:26.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:27.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:27.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:27.117 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:27.127 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:27.230 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:27.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:27.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:27.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:27.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:27.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:27.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:27.622 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:27.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:27.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.567535+0000 mon.a (mon.0) 583 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.585718+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.587496+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.590362+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.590506+0000 mon.a (mon.0) 584 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-09T04:28:27.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[77063]: cluster 2023-10-09T04:28:26.591068+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.567535+0000 mon.a (mon.0) 583 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.585718+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.587496+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.590362+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.590506+0000 mon.a (mon.0) 584 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-09T04:28:27.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:27 smithi082 bash[79733]: cluster 2023-10-09T04:28:26.591068+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.753 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:27.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.567535+0000 mon.a (mon.0) 583 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.585718+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.587496+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.590362+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.590506+0000 mon.a (mon.0) 584 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-09T04:28:27.786 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:27 smithi153 bash[59565]: cluster 2023-10-09T04:28:26.591068+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:28:27.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:27.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:27.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:27.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:27.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 16 97175 97159 84.3231 19.375 0.00498326 0.0118535 2023-10-09T04:28:28.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:28.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:28.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:28.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:28.294 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:28.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:28.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:28.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[67042]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T04:28:28.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[67190]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T04:28:28.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:28.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:28.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:28.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:28.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:28.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:28.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:28.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:28.837 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.374654+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v362: 89 pgs: 9 peering, 10 stale+active+clean, 70 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:28:28.837 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.583496+0000 mon.a (mon.0) 585 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:28:28.837 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.591559+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.837 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.592510+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.592937+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.593485+0000 mon.a (mon.0) 586 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.595174+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.596209+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.597229+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.599427+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[77063]: cluster 2023-10-09T04:28:27.599956+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.838 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.374654+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v362: 89 pgs: 9 peering, 10 stale+active+clean, 70 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.583496+0000 mon.a (mon.0) 585 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.591559+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.592510+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.592937+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.593485+0000 mon.a (mon.0) 586 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.595174+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.596209+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.597229+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.599427+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:28 smithi082 bash[79733]: cluster 2023-10-09T04:28:27.599956+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:28.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.374654+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v362: 89 pgs: 9 peering, 10 stale+active+clean, 70 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.583496+0000 mon.a (mon.0) 585 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.591559+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.592510+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.592937+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.593485+0000 mon.a (mon.0) 586 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.595174+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.596209+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.597229+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.599427+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:28 smithi153 bash[59565]: cluster 2023-10-09T04:28:27.599956+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T04:28:28.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 98177 98161 84.0259 62.625 0.00599209 0.0118936 2023-10-09T04:28:28.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:28.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:29.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:29.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:29.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:29.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:29.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:29.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:29.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:29.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:29.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:29.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:29.549 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:29.549 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:29.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:29.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:29.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:29.766 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:29] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-09T04:28:29.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:29.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:29.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:29.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:29.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 99120 99104 83.6868 58.9375 0.0183435 0.0119442 2023-10-09T04:28:30.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:30.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:30.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:30.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:30.193 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:30.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:30.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:30 smithi153 bash[59565]: audit 2023-10-09T04:28:29.265112+0000 mon.a (mon.0) 587 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:30 smithi153 bash[59565]: audit 2023-10-09T04:28:29.266840+0000 mon.a (mon.0) 588 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:30.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:30 smithi153 bash[59565]: cluster 2023-10-09T04:28:29.375368+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v364: 89 pgs: 2 active+undersized+degraded, 9 peering, 6 stale+active+clean, 6 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6157/289827 objects degraded (2.124%) 2023-10-09T04:28:30.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:30.432 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:30.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[77063]: audit 2023-10-09T04:28:29.265112+0000 mon.a (mon.0) 587 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[77063]: audit 2023-10-09T04:28:29.266840+0000 mon.a (mon.0) 588 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[77063]: cluster 2023-10-09T04:28:29.375368+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v364: 89 pgs: 2 active+undersized+degraded, 9 peering, 6 stale+active+clean, 6 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6157/289827 objects degraded (2.124%) 2023-10-09T04:28:30.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[79733]: audit 2023-10-09T04:28:29.265112+0000 mon.a (mon.0) 587 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[79733]: audit 2023-10-09T04:28:29.266840+0000 mon.a (mon.0) 588 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:30 smithi082 bash[79733]: cluster 2023-10-09T04:28:29.375368+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v364: 89 pgs: 2 active+undersized+degraded, 9 peering, 6 stale+active+clean, 6 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6157/289827 objects degraded (2.124%) 2023-10-09T04:28:30.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:30.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:30.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:30.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:30.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:30.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:30.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:30.958 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 100201 100185 83.4718 67.5625 0.00628546 0.0119748 2023-10-09T04:28:30.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T04:28:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:31.079 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:28:31.079 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T04:28:31.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:31.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:31.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:31.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:31.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:31.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:31.524 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:31 smithi082 bash[77063]: cluster 2023-10-09T04:28:30.260078+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: Degraded data redundancy: 6157/289827 objects degraded (2.124%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:31.525 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:31 smithi082 bash[79733]: cluster 2023-10-09T04:28:30.260078+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: Degraded data redundancy: 6157/289827 objects degraded (2.124%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:31.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:31.655 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.663 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:31 smithi153 bash[59565]: cluster 2023-10-09T04:28:30.260078+0000 mon.a (mon.0) 589 : cluster 3 Health check failed: Degraded data redundancy: 6157/289827 objects degraded (2.124%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:31.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:31.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:31.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:31.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:31.957 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 101192 101176 83.1884 61.9375 0.0166991 0.0120161 2023-10-09T04:28:31.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:31.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:32.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:32.090 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:32.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:28:32.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:32.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:32.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:32.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:32.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:32.478 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:32 smithi153 bash[59565]: cluster 2023-10-09T04:28:31.376398+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v365: 89 pgs: 10 active+undersized+degraded, 9 peering, 11 active+undersized, 59 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 31049/294195 objects degraded (10.554%) 2023-10-09T04:28:32.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:32 smithi082 bash[77063]: cluster 2023-10-09T04:28:31.376398+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v365: 89 pgs: 10 active+undersized+degraded, 9 peering, 11 active+undersized, 59 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 31049/294195 objects degraded (10.554%) 2023-10-09T04:28:32.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:32 smithi082 bash[79733]: cluster 2023-10-09T04:28:31.376398+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v365: 89 pgs: 10 active+undersized+degraded, 9 peering, 11 active+undersized, 59 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 31049/294195 objects degraded (10.554%) 2023-10-09T04:28:32.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:32.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:32.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:32.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:32.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:32.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:32.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:32.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:32.958 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 16 102222 102206 82.944 64.375 0.00467565 0.0120507 2023-10-09T04:28:33.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:33.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:33.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:33.159 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:33.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:33.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:33.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.430 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:33.539 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:33.566 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:33.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:33 smithi153 bash[59565]: cluster 2023-10-09T04:28:33.428810+0000 mon.a (mon.0) 590 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:28:33.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:33.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:33 smithi082 bash[77063]: cluster 2023-10-09T04:28:33.428810+0000 mon.a (mon.0) 590 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:28:33.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:33 smithi082 bash[79733]: cluster 2023-10-09T04:28:33.428810+0000 mon.a (mon.0) 590 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:28:33.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:33.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:33.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.958 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 103197 103181 82.6618 60.9375 0.00766713 0.0120904 2023-10-09T04:28:33.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:33.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:34.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.103 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:34.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:34.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.368 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:34.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:34.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:34.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:34.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:34.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:34 smithi082 bash[79733]: cluster 2023-10-09T04:28:33.377291+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v366: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 56312/299790 objects degraded (18.784%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T04:28:34.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:34 smithi082 bash[77063]: cluster 2023-10-09T04:28:33.377291+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v366: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 56312/299790 objects degraded (18.784%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T04:28:34.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:34.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:34.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:34.902 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:34.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:34.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:34 smithi153 bash[59565]: cluster 2023-10-09T04:28:33.377291+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v366: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 56312/299790 objects degraded (18.784%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T04:28:34.959 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 104196 104180 82.4042 62.4375 0.0142679 0.0121301 2023-10-09T04:28:35.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:35.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:35.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.327 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:35.454 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:28:35.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:35.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:35.493 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:28:35.554 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:28:35.575 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.594 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:28:35.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.636 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:28:35.666 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:28:35.707 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:28:35.707 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.707 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.707 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.708 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.708 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:28:35.708 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.708 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:35.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.724 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:35 smithi082 bash[77063]: audit 2023-10-09T04:28:34.960977+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:35.724 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:35 smithi082 bash[77063]: audit 2023-10-09T04:28:35.382535+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.82:0/2331116465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:35.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:35 smithi082 bash[79733]: audit 2023-10-09T04:28:34.960977+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:35.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:35 smithi082 bash[79733]: audit 2023-10-09T04:28:35.382535+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.82:0/2331116465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:35.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:35.847 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:35 smithi153 bash[59565]: audit 2023-10-09T04:28:34.960977+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:35.847 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:35 smithi153 bash[59565]: audit 2023-10-09T04:28:35.382535+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.82:0/2331116465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:35.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:35.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:35.959 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:28:35.959238+0000 min lat: 0.00135884 max lat: 1.06331 avg lat: 0.0121828 2023-10-09T04:28:35.960 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:28:35.960 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 105056 105040 82.046 53.75 0.00894314 0.0121828 2023-10-09T04:28:35.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:36.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:36.080 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:28:36.081 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:28:36.081 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:28:36.081 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-10-09T04:28:36.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-10-09T04:28:36.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:36.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:36.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:36.443 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:36.523 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:28:36.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:36.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[77063]: cluster 2023-10-09T04:28:35.378255+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v367: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 57855/308019 objects degraded (18.783%); 7.3 KiB/s, 0 objects/s recovering 2023-10-09T04:28:36.552 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[77063]: cluster 2023-10-09T04:28:35.485699+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 57855/308019 objects degraded (18.783%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:36.552 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[77063]: audit 2023-10-09T04:28:35.891990+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:36.552 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[79733]: cluster 2023-10-09T04:28:35.378255+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v367: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 57855/308019 objects degraded (18.783%); 7.3 KiB/s, 0 objects/s recovering 2023-10-09T04:28:36.552 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[79733]: cluster 2023-10-09T04:28:35.485699+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 57855/308019 objects degraded (18.783%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:36.552 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:36 smithi082 bash[79733]: audit 2023-10-09T04:28:35.891990+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:36.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:36 smithi153 bash[59565]: cluster 2023-10-09T04:28:35.378255+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v367: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 57855/308019 objects degraded (18.783%); 7.3 KiB/s, 0 objects/s recovering 2023-10-09T04:28:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:36 smithi153 bash[59565]: cluster 2023-10-09T04:28:35.485699+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 57855/308019 objects degraded (18.783%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:36.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:36 smithi153 bash[59565]: audit 2023-10-09T04:28:35.891990+0000 mon.a (mon.0) 593 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:36.692 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:36.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:36.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:36.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:36.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:36.959 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 105983 105967 81.7483 57.9375 0.0114414 0.0122275 2023-10-09T04:28:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:28:36.992 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:28:36.994 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:37.100 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:37.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:37.236 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:37.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:37.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:37.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:37.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:37.502 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:37.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:37.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.500256+0000 mon.a (mon.0) 594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: audit 2023-10-09T04:28:36.517910+0000 mon.a (mon.0) 595 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.517988+0000 mon.a (mon.0) 596 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.521160+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.524162+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.537694+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.637 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:37 smithi153 bash[59565]: cluster 2023-10-09T04:28:36.539179+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:37.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:37.775 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.500256+0000 mon.a (mon.0) 594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:37.775 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: audit 2023-10-09T04:28:36.517910+0000 mon.a (mon.0) 595 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:37.775 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.517988+0000 mon.a (mon.0) 596 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:28:37.775 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.521160+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.524162+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.537694+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[77063]: cluster 2023-10-09T04:28:36.539179+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.500256+0000 mon.a (mon.0) 594 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: audit 2023-10-09T04:28:36.517910+0000 mon.a (mon.0) 595 : audit 1 from='client.? 172.21.15.82:0/2244028434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.517988+0000 mon.a (mon.0) 596 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.521160+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.524162+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.537694+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:37 smithi082 bash[79733]: cluster 2023-10-09T04:28:36.539179+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T04:28:37.820 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:37.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:37.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:37.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:37.959 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 106989 106973 81.5181 62.875 0.0116224 0.012262 2023-10-09T04:28:38.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.089 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:38.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:38.243 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:38.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:38.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:38.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:38.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:38.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:38.641 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:38 smithi082 bash[77063]: cluster 2023-10-09T04:28:37.378778+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v369: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 59074/314307 objects degraded (18.795%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T04:28:38.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:38 smithi082 bash[79733]: cluster 2023-10-09T04:28:37.378778+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v369: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 59074/314307 objects degraded (18.795%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T04:28:38.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:38.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:38 smithi153 bash[59565]: cluster 2023-10-09T04:28:37.378778+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v369: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 59074/314307 objects degraded (18.795%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T04:28:38.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:38.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:38.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:38.960 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 16 107962 107946 81.2686 60.8125 0.00894932 0.012299 2023-10-09T04:28:38.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:39.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:39.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:39.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:39.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:39.246 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:39.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:39.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:39.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:39.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:39.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:28:39.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:39.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:39.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.904 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:39] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-09T04:28:39.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:39.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:39.960 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 108818 108802 80.938 53.5 0.00931342 0.0123494 2023-10-09T04:28:40.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:40.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:40.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:40.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.354 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:40.458 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:40 smithi153 bash[59565]: cluster 2023-10-09T04:28:39.379512+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v370: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 59354/316938 objects degraded (18.727%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:40.458 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:40 smithi153 bash[59565]: audit 2023-10-09T04:28:39.456149+0000 mon.a (mon.0) 597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:40.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:40.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:40.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:40.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.602 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:40 smithi082 bash[77063]: cluster 2023-10-09T04:28:39.379512+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v370: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 59354/316938 objects degraded (18.727%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:40.602 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:40 smithi082 bash[77063]: audit 2023-10-09T04:28:39.456149+0000 mon.a (mon.0) 597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:40.602 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:40 smithi082 bash[79733]: cluster 2023-10-09T04:28:39.379512+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v370: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 59354/316938 objects degraded (18.727%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:40.602 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:40 smithi082 bash[79733]: audit 2023-10-09T04:28:39.456149+0000 mon.a (mon.0) 597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:40.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:40.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:40.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:40.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:40.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:40.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:40.961 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 109645 109629 80.5925 51.6875 0.0267753 0.0124024 2023-10-09T04:28:41.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:41.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:41.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:41.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:41.302 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:41.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.386 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:41.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:41.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:41.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:41.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:41.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:41.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:41.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.941 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:41.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:41.961 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 110455 110439 80.244 50.625 0.0437727 0.0124564 2023-10-09T04:28:42.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:42.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:42.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:42.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:42.297 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:42.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:41.374833+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: cluster 2023-10-09T04:28:41.380235+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v371: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 60506/322287 objects degraded (18.774%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:41.387363+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:41.752908+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:41.759756+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:42.337351+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:42.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:42.342332+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:42.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[77063]: audit 2023-10-09T04:28:42.351490+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:41.374833+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: cluster 2023-10-09T04:28:41.380235+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v371: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 60506/322287 objects degraded (18.774%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:41.387363+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:41.752908+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:41.759756+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.434 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:42.337351+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:42.435 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:42.342332+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:42.435 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:42 smithi153 bash[59565]: audit 2023-10-09T04:28:42.351490+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:42.554 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:42.583 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:42.686 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:42.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:41.374833+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: cluster 2023-10-09T04:28:41.380235+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v371: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 60506/322287 objects degraded (18.774%); 6.4 KiB/s, 0 objects/s recovering 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:41.387363+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:41.752908+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:41.759756+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:42.337351+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:42.342332+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:42 smithi082 bash[79733]: audit 2023-10-09T04:28:42.351490+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:42.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:42.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:42.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:42.961 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 111331 111315 79.951 54.75 0.00468278 0.012501 2023-10-09T04:28:42.977 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.003 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:43.083 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:111400 5.4 e116/114 5:26d1996c:::benchmark_data_smithi082_106363_object111399:head v 116'6908, mlcod=116'6905)", 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.057511+0000", 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.022987411999999999, 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023026571999999999, 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.084 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057511+0000", 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057511+0000", 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.085 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057510+0000", 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057546+0000", 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6171000000000003e-05 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.086 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057560+0000", 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4656e-05 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057570+0000", 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4220000000000006e-06 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057621+0000", 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1554999999999997e-05 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.057650+0000", 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8232e-05 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:111411 5.4 e116/114 5:22cc1f16:::benchmark_data_smithi082_106363_object111410:head v 116'6909, mlcod=116'6905)", 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.064709+0000", 2023-10-09T04:28:43.088 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.015789109999999999, 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.015899686, 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064709+0000", 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.089 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064709+0000", 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064708+0000", 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064736+0000", 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7679e-05 2023-10-09T04:28:43.090 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064745+0000", 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4059999999999994e-06 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.064751+0000", 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8490000000000002e-06 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.091 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069162+0000", 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0044114569999999997 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069178+0000", 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5233e-05 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.092 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:111412 5.4 e116/114 5:27b4dc20:::benchmark_data_smithi082_106363_object111411:head v 116'6910, mlcod=116'6905)", 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.069148+0000", 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.011350258, 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011530301999999999, 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:28:43.093 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069148+0000", 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069148+0000", 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.094 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069147+0000", 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069182+0000", 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4335000000000002e-05 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.095 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069189+0000", 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1539999999999996e-06 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069194+0000", 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9819999999999999e-06 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.096 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074560+0000", 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053661500000000001 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074582+0000", 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2055000000000001e-05 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.097 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:111417 5.8 5:1107f8f2:::benchmark_data_smithi082_106363_object111416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.069705+0000", 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.010793445, 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011086491, 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:28:43.098 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 111417 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069705+0000", 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.099 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069705+0000", 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069703+0000", 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.100 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069734+0000", 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0800000000000003e-05 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069745+0000", 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0772e-05 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.101 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069752+0000", 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6520000000000006e-06 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.069814+0000", 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1884000000000006e-05 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.102 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070008+0000", 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019381699999999999 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070121+0000", 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011290000000000001 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.103 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074552+0000", 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0044308689999999996 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.080563+0000", 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060115000000000004 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.104 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:111419 5.4 e116/114 5:2e4ba577:::benchmark_data_smithi082_106363_object111418:head v 116'6911, mlcod=116'6906)", 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.070747+0000", 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0097509529999999997, 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010370717, 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.105 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070747+0000", 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070747+0000", 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.106 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070746+0000", 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070807+0000", 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1043999999999994e-05 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070818+0000", 2023-10-09T04:28:43.107 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0482000000000001e-05 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.070824+0000", 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6630000000000004e-06 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.078487+0000", 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.007663001 2023-10-09T04:28:43.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.078512+0000", 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4899999999999999e-05 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:111421 5.b 5:d6f2861b:::benchmark_data_smithi082_106363_object111420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:43.074641+0000", 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0058573419999999998, 2023-10-09T04:28:43.109 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065160920000000002, 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 111421 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074641+0000", 2023-10-09T04:28:43.110 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074641+0000", 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074639+0000", 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.111 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074661+0000", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1902000000000001e-05 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074672+0000", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0377000000000001e-05 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074679+0000", 2023-10-09T04:28:43.112 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4449999999999999e-06 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074725+0000", 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5867e-05 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.074847+0000", 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000121648 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.075018+0000", 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000171588 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:43.078926+0000", 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039077230000000001 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:28:43.114 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:28:43.115 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 6 2023-10-09T04:28:43.115 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:28:43.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:43.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:43.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:43.357 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:43.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:43 smithi082 bash[77063]: cluster 2023-10-09T04:28:42.371436+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 60506/322287 objects degraded (18.774%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:43.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:43.525 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:43.534 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:43 smithi153 bash[59565]: cluster 2023-10-09T04:28:42.371436+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 60506/322287 objects degraded (18.774%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:43.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:43.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:43.662 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:43 smithi082 bash[79733]: cluster 2023-10-09T04:28:42.371436+0000 mon.a (mon.0) 605 : cluster 3 Health check update: Degraded data redundancy: 60506/322287 objects degraded (18.774%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:43.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:43.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:43.828 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:43.931 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:43.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 112304 112288 79.7321 60.8125 0.01079 0.0125368 2023-10-09T04:28:43.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:43.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:44.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:44.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:44.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:44.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:44.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:44.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:44.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:44.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:44 smithi153 bash[59565]: cluster 2023-10-09T04:28:43.381312+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v372: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 61680/328323 objects degraded (18.786%) 2023-10-09T04:28:44.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:44 smithi153 bash[59565]: audit 2023-10-09T04:28:44.261978+0000 mon.a (mon.0) 606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:44.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:44 smithi153 bash[59565]: audit 2023-10-09T04:28:44.263230+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:44.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:44.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:44.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[77063]: cluster 2023-10-09T04:28:43.381312+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v372: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 61680/328323 objects degraded (18.786%) 2023-10-09T04:28:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[77063]: audit 2023-10-09T04:28:44.261978+0000 mon.a (mon.0) 606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[77063]: audit 2023-10-09T04:28:44.263230+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[79733]: cluster 2023-10-09T04:28:43.381312+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v372: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 61680/328323 objects degraded (18.786%) 2023-10-09T04:28:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[79733]: audit 2023-10-09T04:28:44.261978+0000 mon.a (mon.0) 606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:28:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:44 smithi082 bash[79733]: audit 2023-10-09T04:28:44.263230+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:44.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:44.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:44.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:44.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:44.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:44.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:44.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 16 113304 113288 79.5385 62.5 0.00610089 0.0125674 2023-10-09T04:28:45.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:45.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:45.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:45.172 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:45.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:45.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:45.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:45.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:45.678 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:45.704 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:45.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:45.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:45.958 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:45.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 90.008 2023-10-09T04:28:45.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 114228 2023-10-09T04:28:45.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:28:45.963 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 79.318 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 30.9159 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 188.438 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 19.375 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 1269 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 494.655 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 3015 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 310 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.0126032 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.0156121 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 1.06331 2023-10-09T04:28:45.964 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00135884 2023-10-09T04:28:45.970 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:28:45.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:45.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:46.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:28:46.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:46.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:46.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:46.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:46.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:46.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:46.598 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:46.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.686 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:46 smithi082 bash[77063]: cluster 2023-10-09T04:28:45.382048+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v373: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 63074/335958 objects degraded (18.774%) 2023-10-09T04:28:46.686 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:46 smithi082 bash[79733]: cluster 2023-10-09T04:28:45.382048+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v373: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 63074/335958 objects degraded (18.774%) 2023-10-09T04:28:46.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:46.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:46.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:46 smithi153 bash[59565]: cluster 2023-10-09T04:28:45.382048+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v373: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 63074/335958 objects degraded (18.774%) 2023-10-09T04:28:46.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:46.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:46.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:47.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:47.215 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:47.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:47.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:47.476 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:47.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:47 smithi082 bash[77063]: cluster 2023-10-09T04:28:47.433885+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 63737/339564 objects degraded (18.770%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:47 smithi082 bash[79733]: cluster 2023-10-09T04:28:47.433885+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 63737/339564 objects degraded (18.770%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:47.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.760 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:47 smithi153 bash[59565]: cluster 2023-10-09T04:28:47.433885+0000 mon.a (mon.0) 608 : cluster 3 Health check update: Degraded data redundancy: 63737/339564 objects degraded (18.770%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:47.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:47.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:47.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:47.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:47.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:48.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.039 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:48.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:48.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:48.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:48.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:48.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:48.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:48.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:48.494 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:48 smithi082 bash[77063]: cluster 2023-10-09T04:28:47.382702+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v374: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 63 MiB/s wr, 1.04k op/s; 63737/339564 objects degraded (18.770%) 2023-10-09T04:28:48.495 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:48 smithi082 bash[79733]: cluster 2023-10-09T04:28:47.382702+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v374: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 63 MiB/s wr, 1.04k op/s; 63737/339564 objects degraded (18.770%) 2023-10-09T04:28:48.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:48.545 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:48 smithi153 bash[59565]: cluster 2023-10-09T04:28:47.382702+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v374: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 63 MiB/s wr, 1.04k op/s; 63737/339564 objects degraded (18.770%) 2023-10-09T04:28:48.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:48.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:48.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:48.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:48.817 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:48.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:48.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:48.968 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:48.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:48.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:49.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:49.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:49.206 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:49.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:49.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:49.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:49.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:49.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:49.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:49.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:49.598 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:49.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:49.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:49.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:49.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:49.881 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:49] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-09T04:28:49.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:49.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:49.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:50.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:50.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:50.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:50.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:50.406 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:50.517 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.3 primary-affinity to 0 (802) 2023-10-09T04:28:50.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:50 smithi153 bash[59565]: cluster 2023-10-09T04:28:49.383365+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v375: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 841 op/s; 63728/339342 objects degraded (18.780%) 2023-10-09T04:28:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:50 smithi153 bash[59565]: audit 2023-10-09T04:28:49.510805+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.82:0/2925317416' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:50 smithi153 bash[59565]: audit 2023-10-09T04:28:49.511920+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:50.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:50.835 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:50.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:50.950 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[77063]: cluster 2023-10-09T04:28:49.383365+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v375: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 841 op/s; 63728/339342 objects degraded (18.780%) 2023-10-09T04:28:50.951 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[77063]: audit 2023-10-09T04:28:49.510805+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.82:0/2925317416' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.951 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[77063]: audit 2023-10-09T04:28:49.511920+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.951 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[79733]: cluster 2023-10-09T04:28:49.383365+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v375: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 841 op/s; 63728/339342 objects degraded (18.780%) 2023-10-09T04:28:50.951 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[79733]: audit 2023-10-09T04:28:49.510805+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.82:0/2925317416' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.951 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:50 smithi082 bash[79733]: audit 2023-10-09T04:28:49.511920+0000 mon.a (mon.0) 609 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T04:28:50.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:50.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:51.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:51.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:51.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:51.195 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:51.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:51.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:51.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:51.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:51.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:51.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:51.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:51.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:51.725 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:51.739 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.496355+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: audit 2023-10-09T04:28:50.499401+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.499498+0000 mon.a (mon.0) 611 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.501092+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.504934+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.508004+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.510320+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.510351+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.510556+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.512031+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.513432+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.514815+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[77063]: cluster 2023-10-09T04:28:50.515559+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.496355+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: audit 2023-10-09T04:28:50.499401+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.499498+0000 mon.a (mon.0) 611 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.501092+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.741 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.504934+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.508004+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.510320+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.510351+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.510556+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.512031+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.513432+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.514815+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:51 smithi082 bash[79733]: cluster 2023-10-09T04:28:50.515559+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:51.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.496355+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: audit 2023-10-09T04:28:50.499401+0000 mon.a (mon.0) 610 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.499498+0000 mon.a (mon.0) 611 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.501092+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.504934+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.508004+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.510320+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.510351+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.878 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.510556+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.879 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.512031+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.879 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.513432+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.879 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.514815+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.879 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:51 smithi153 bash[59565]: cluster 2023-10-09T04:28:50.515559+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T04:28:51.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:51.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:52.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:52.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:52.152 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:52.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:52.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:52.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:52.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:52.541 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.384092+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v377: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 925 op/s; 62848/335493 objects degraded (18.733%) 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.516705+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.516766+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.518308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.519528+0000 mon.a (mon.0) 612 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.520158+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.520438+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:52 smithi153 bash[59565]: cluster 2023-10-09T04:28:51.523776+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:52.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:52.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:52.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.384092+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v377: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 925 op/s; 62848/335493 objects degraded (18.733%) 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.516705+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.516766+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.518308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.519528+0000 mon.a (mon.0) 612 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.520158+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.520438+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.938 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[77063]: cluster 2023-10-09T04:28:51.523776+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.938 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.384092+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v377: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 925 op/s; 62848/335493 objects degraded (18.733%) 2023-10-09T04:28:52.938 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.516705+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.516766+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.518308+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.519528+0000 mon.a (mon.0) 612 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.520158+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.520438+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:52 smithi082 bash[79733]: cluster 2023-10-09T04:28:51.523776+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T04:28:52.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:52.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:52.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:53.012 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:53.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:53.084 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:53.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:53.220 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:53.267 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:28:53.302 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:28:53.326 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:53.349 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:28:53.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:53.382 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:28:53.420 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:28:53.462 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:28:53.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.495 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:28:53.495 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.496 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:28:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:53.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:53 smithi153 bash[59565]: audit 2023-10-09T04:28:53.222670+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.82:0/1800202102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:53.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:53.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.750 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:53.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:53.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:53.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:53.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:53 smithi082 bash[77063]: audit 2023-10-09T04:28:53.222670+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.82:0/1800202102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:53.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:53 smithi082 bash[79733]: audit 2023-10-09T04:28:53.222670+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.82:0/1800202102' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:54.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:54.033 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:54.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:54.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:54.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:54.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:54.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:54.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:54.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:54.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:54.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:54.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:54.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:54.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:54 smithi082 bash[79733]: cluster 2023-10-09T04:28:53.384663+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v379: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 802 op/s; 61880/330309 objects degraded (18.734%) 2023-10-09T04:28:54.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:54 smithi082 bash[79733]: cluster 2023-10-09T04:28:53.514568+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 61880/330309 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:54.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:54 smithi082 bash[77063]: cluster 2023-10-09T04:28:53.384663+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v379: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 802 op/s; 61880/330309 objects degraded (18.734%) 2023-10-09T04:28:54.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:54 smithi082 bash[77063]: cluster 2023-10-09T04:28:53.514568+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 61880/330309 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:54.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:54.825 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:54 smithi153 bash[59565]: cluster 2023-10-09T04:28:53.384663+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v379: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 802 op/s; 61880/330309 objects degraded (18.734%) 2023-10-09T04:28:54.825 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:54 smithi153 bash[59565]: cluster 2023-10-09T04:28:53.514568+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 61880/330309 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:54.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:54.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:54.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:54.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:54.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:55.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:55.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:55.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:55.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:55.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:55.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:55.383 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:55.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:55.508 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:55.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:55.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:55.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:55.748 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:55.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:55.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:55.902 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:55.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:55.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:56.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:56.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:56.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:56.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:56.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:56.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:28:56.469 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:28:56.540 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:56.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:56.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:56.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:56.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:56.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.828 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:56 smithi153 bash[59565]: cluster 2023-10-09T04:28:55.385305+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v380: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.2 MiB/s wr, 823 op/s; 60725/323796 objects degraded (18.754%) 2023-10-09T04:28:56.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:56.845 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:56 smithi082 bash[77063]: cluster 2023-10-09T04:28:55.385305+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v380: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.2 MiB/s wr, 823 op/s; 60725/323796 objects degraded (18.754%) 2023-10-09T04:28:56.846 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:56 smithi082 bash[79733]: cluster 2023-10-09T04:28:55.385305+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v380: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.2 MiB/s wr, 823 op/s; 60725/323796 objects degraded (18.754%) 2023-10-09T04:28:56.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:56.980 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:56.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:57.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:57.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:57.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:57.235 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:57.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:57.368 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:57.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:57.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:57.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:57.500 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:28:57.500 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:28:57.501 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.8167226012025871 2023-10-09T04:28:57.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 0.8167226012025871 2023-10-09T04:28:57.596 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:28:57.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:57.633 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:57 smithi082 bash[77063]: audit 2023-10-09T04:28:57.122375+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.82:0/209572865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.633 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:57 smithi082 bash[77063]: audit 2023-10-09T04:28:57.123380+0000 mon.a (mon.0) 614 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.633 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:57 smithi082 bash[79733]: audit 2023-10-09T04:28:57.122375+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.82:0/209572865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.634 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:57 smithi082 bash[79733]: audit 2023-10-09T04:28:57.123380+0000 mon.a (mon.0) 614 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:57.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:57.771 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:57.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:57.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:57 smithi153 bash[59565]: audit 2023-10-09T04:28:57.122375+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.82:0/209572865' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:57 smithi153 bash[59565]: audit 2023-10-09T04:28:57.123380+0000 mon.a (mon.0) 614 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:57.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:57.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:58.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:58.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:58.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:58.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:58.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:58.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:58.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:58.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:58.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:28:58.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:58.572 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:58.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.385989+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v381: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 968 op/s; 59545/317694 objects degraded (18.743%) 2023-10-09T04:28:58.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.537745+0000 mon.a (mon.0) 615 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.548320+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: audit 2023-10-09T04:28:57.557053+0000 mon.a (mon.0) 616 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.557264+0000 mon.a (mon.0) 617 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.557991+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[77063]: cluster 2023-10-09T04:28:57.560726+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.385989+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v381: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 968 op/s; 59545/317694 objects degraded (18.743%) 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.537745+0000 mon.a (mon.0) 615 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.548320+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: audit 2023-10-09T04:28:57.557053+0000 mon.a (mon.0) 616 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:58.672 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.557264+0000 mon.a (mon.0) 617 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T04:28:58.672 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.557991+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.672 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:58 smithi082 bash[79733]: cluster 2023-10-09T04:28:57.560726+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.385989+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v381: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 968 op/s; 59545/317694 objects degraded (18.743%) 2023-10-09T04:28:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.537745+0000 mon.a (mon.0) 615 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.548320+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: audit 2023-10-09T04:28:57.557053+0000 mon.a (mon.0) 616 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.557264+0000 mon.a (mon.0) 617 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-09T04:28:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.557991+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:58 smithi153 bash[59565]: cluster 2023-10-09T04:28:57.560726+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T04:28:58.678 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:58.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:58.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:58.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:58.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:58.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:58.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:58.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:59.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:59.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:59.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:59.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:59.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:59.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:59.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:59.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:59.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.615 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:59 smithi082 bash[79733]: audit 2023-10-09T04:28:59.257830+0000 mon.a (mon.0) 618 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:59.615 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:28:59 smithi082 bash[79733]: cluster 2023-10-09T04:28:59.428930+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 59545/317694 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:59.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:59.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:59.776 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:59.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:59 smithi153 bash[59565]: audit 2023-10-09T04:28:59.257830+0000 mon.a (mon.0) 618 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:59.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:28:59 smithi153 bash[59565]: cluster 2023-10-09T04:28:59.428930+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 59545/317694 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:59.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:28:59.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:59 smithi082 bash[77063]: audit 2023-10-09T04:28:59.257830+0000 mon.a (mon.0) 618 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:28:59 smithi082 bash[77063]: cluster 2023-10-09T04:28:59.428930+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 59545/317694 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:28:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:28:59] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-09T04:29:00.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:00.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:00.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:00.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:00.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:00.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:00.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:00.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:00.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:00.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:00.527 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:00.527 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:00.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:00.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:00.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:00.707 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:00 smithi082 bash[77063]: cluster 2023-10-09T04:28:59.386502+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v383: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 59265/315069 objects degraded (18.810%) 2023-10-09T04:29:00.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:00 smithi082 bash[79733]: cluster 2023-10-09T04:28:59.386502+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v383: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 59265/315069 objects degraded (18.810%) 2023-10-09T04:29:00.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:00.840 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:00.846 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:00 smithi153 bash[59565]: cluster 2023-10-09T04:28:59.386502+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v383: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 59265/315069 objects degraded (18.810%) 2023-10-09T04:29:00.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:00.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:00.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:00.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.011 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:01.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:01.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:01.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:01.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:01.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:01.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:01.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:01.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:29:01.649 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:01.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:01.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:01.840 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:29:01.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:01.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:01.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:01.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:01.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:02.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:02.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:02.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:02.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:02.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:02.393 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:02.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:02.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:02.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:02.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:02.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.773 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:02 smithi082 bash[77063]: cluster 2023-10-09T04:29:01.387262+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v384: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 58118/309930 objects degraded (18.752%) 2023-10-09T04:29:02.773 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:02 smithi082 bash[79733]: cluster 2023-10-09T04:29:01.387262+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v384: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 58118/309930 objects degraded (18.752%) 2023-10-09T04:29:02.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:02.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:02.905 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:02 smithi153 bash[59565]: cluster 2023-10-09T04:29:01.387262+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v384: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 58118/309930 objects degraded (18.752%) 2023-10-09T04:29:02.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:02.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:02.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:03.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:03.060 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:03.077 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:03.078 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:03.165 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:03.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:03.296 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:03.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:03.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:03.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:03.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:03.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:03.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:03.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:03.822 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:03.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:03.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:03.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:03.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:04.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:04.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:04.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:04.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:04.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:04.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:04.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:04.524 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:04.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:04.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:04.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:04.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:04 smithi153 bash[59565]: cluster 2023-10-09T04:29:03.387966+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v385: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 56914/303654 objects degraded (18.743%) 2023-10-09T04:29:04.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:04 smithi153 bash[59565]: cluster 2023-10-09T04:29:04.430708+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 56914/303654 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:04.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:04.776 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:04 smithi082 bash[77063]: cluster 2023-10-09T04:29:03.387966+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v385: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 56914/303654 objects degraded (18.743%) 2023-10-09T04:29:04.777 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:04 smithi082 bash[77063]: cluster 2023-10-09T04:29:04.430708+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 56914/303654 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:04.777 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:04 smithi082 bash[79733]: cluster 2023-10-09T04:29:03.387966+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v385: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 56914/303654 objects degraded (18.743%) 2023-10-09T04:29:04.777 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:04 smithi082 bash[79733]: cluster 2023-10-09T04:29:04.430708+0000 mon.a (mon.0) 620 : cluster 3 Health check update: Degraded data redundancy: 56914/303654 objects degraded (18.743%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:04.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:04.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:04.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:04.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:05.032 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:05.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:05.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:05.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:05.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:05.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:05.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:05.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:05.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:05.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:05.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:05.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:05.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:05.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:05.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:05.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:05.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:05.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:06.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:06.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:06.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:06.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:06.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:06.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:06.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:06 smithi153 bash[59565]: cluster 2023-10-09T04:29:05.388755+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v386: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 55568/296439 objects degraded (18.745%) 2023-10-09T04:29:06.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.682 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:06.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:06.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:06.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:06.917 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:06.926 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:06 smithi082 bash[77063]: cluster 2023-10-09T04:29:05.388755+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v386: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 55568/296439 objects degraded (18.745%) 2023-10-09T04:29:06.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:06 smithi082 bash[79733]: cluster 2023-10-09T04:29:05.388755+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v386: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 55568/296439 objects degraded (18.745%) 2023-10-09T04:29:06.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:06.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:07.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:07.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:07.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:07.333 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:07.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:07.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:07.499 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:07.550 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:07.554 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:07.606 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:07.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:07.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:07.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:07.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:07.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:08.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:08.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:08.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:08.176 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:08.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:08.314 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:08.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:08.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:08.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:08.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:08.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:08 smithi082 bash[77063]: cluster 2023-10-09T04:29:07.390898+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v387: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 54511/290946 objects degraded (18.736%) 2023-10-09T04:29:08.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:08 smithi082 bash[79733]: cluster 2023-10-09T04:29:07.390898+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v387: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 54511/290946 objects degraded (18.736%) 2023-10-09T04:29:08.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:08.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:08 smithi153 bash[59565]: cluster 2023-10-09T04:29:07.390898+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v387: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 54511/290946 objects degraded (18.736%) 2023-10-09T04:29:08.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:08.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:08.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:09.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:09.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:09.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:09.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:09.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:09.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:09.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:09.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:09.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:09.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:09.639 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:09 smithi082 bash[77063]: cluster 2023-10-09T04:29:09.431894+0000 mon.a (mon.0) 621 : cluster 3 Health check update: Degraded data redundancy: 54511/290946 objects degraded (18.736%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:09.639 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:09 smithi082 bash[79733]: cluster 2023-10-09T04:29:09.431894+0000 mon.a (mon.0) 621 : cluster 3 Health check update: Degraded data redundancy: 54511/290946 objects degraded (18.736%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:09.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:09.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:09.780 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:09 smithi153 bash[59565]: cluster 2023-10-09T04:29:09.431894+0000 mon.a (mon.0) 621 : cluster 3 Health check update: Degraded data redundancy: 54511/290946 objects degraded (18.736%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:09.794 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:09.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:09.934 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:09.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:09.984 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:09] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-09T04:29:10.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:10.065 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:10.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:10.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:10.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:10.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:10.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:10.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:10.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:10.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:10.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:10.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:10.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:10.844 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:10 smithi153 bash[59565]: cluster 2023-10-09T04:29:09.391531+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v388: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 54226/288444 objects degraded (18.799%) 2023-10-09T04:29:10.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:10.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:10.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:10.867 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:10 smithi082 bash[77063]: cluster 2023-10-09T04:29:09.391531+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v388: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 54226/288444 objects degraded (18.799%) 2023-10-09T04:29:10.868 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:10 smithi082 bash[79733]: cluster 2023-10-09T04:29:09.391531+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v388: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 54226/288444 objects degraded (18.799%) 2023-10-09T04:29:10.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:11.002 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:11.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:11.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:11.183 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:29:11.226 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:29:11.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:11.259 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:29:11.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:11.306 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:29:11.346 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:29:11.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:11.389 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:29:11.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:11.433 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:29:11.434 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.435 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:29:11.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:11.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:11.618 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[77063]: audit 2023-10-09T04:29:11.138147+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.82:0/2717771762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.618 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[77063]: audit 2023-10-09T04:29:11.564528+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.82:0/4101325512' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:11.618 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[77063]: audit 2023-10-09T04:29:11.565005+0000 mon.a (mon.0) 622 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:11.620 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 0.816723 (d114) 2023-10-09T04:29:11.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.650 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[79733]: audit 2023-10-09T04:29:11.138147+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.82:0/2717771762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.650 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[79733]: audit 2023-10-09T04:29:11.564528+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.82:0/4101325512' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:11.651 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:11 smithi082 bash[79733]: audit 2023-10-09T04:29:11.565005+0000 mon.a (mon.0) 622 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:11.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:11.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.791 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:11.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:11.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:11 smithi153 bash[59565]: audit 2023-10-09T04:29:11.138147+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.82:0/2717771762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:11 smithi153 bash[59565]: audit 2023-10-09T04:29:11.564528+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.82:0/4101325512' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:11 smithi153 bash[59565]: audit 2023-10-09T04:29:11.565005+0000 mon.a (mon.0) 622 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]: dispatch 2023-10-09T04:29:12.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:12.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:12.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:12.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:12.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:12.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:12.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:12.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:12.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:12.667 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.392291+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v389: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 53096/283422 objects degraded (18.734%) 2023-10-09T04:29:12.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: audit 2023-10-09T04:29:11.608101+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]': finished 2023-10-09T04:29:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.608237+0000 mon.a (mon.0) 624 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T04:29:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.609293+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.616989+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.616992+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:12 smithi153 bash[59565]: cluster 2023-10-09T04:29:11.617121+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:12.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:12.813 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.392291+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v389: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 53096/283422 objects degraded (18.734%) 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: audit 2023-10-09T04:29:11.608101+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]': finished 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.608237+0000 mon.a (mon.0) 624 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.609293+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.616989+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.616992+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[77063]: cluster 2023-10-09T04:29:11.617121+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.392291+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v389: 89 pgs: 16 active+undersized+degraded, 14 active+undersized, 59 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 53096/283422 objects degraded (18.734%) 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: audit 2023-10-09T04:29:11.608101+0000 mon.a (mon.0) 623 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.8167226012025871}]': finished 2023-10-09T04:29:12.814 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.608237+0000 mon.a (mon.0) 624 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-09T04:29:12.815 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.609293+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.815 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.616989+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.815 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.616992+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.815 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:12 smithi082 bash[79733]: cluster 2023-10-09T04:29:11.617121+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T04:29:12.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:12.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:12.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:13.062 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:13.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:13.202 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:13.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:13.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:13.369 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:13.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:13.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:13.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:13.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:13.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:13.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:12.622652+0000 mon.a (mon.0) 625 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:12.622847+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:12.625348+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:12.626165+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:12.628584+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.626489+0000 mon.a (mon.0) 626 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:12.622652+0000 mon.a (mon.0) 625 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T04:29:13.742 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:12.622847+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:12.625348+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:12.626165+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:12.628584+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.743 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:13 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.626489+0000 mon.a (mon.0) 626 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T04:29:13.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:13.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:13.887 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:13.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:13.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:12.622652+0000 mon.a (mon.0) 625 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-10-09T04:29:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:12.622847+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:12.625348+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:12.626165+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:12.628584+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T04:29:13.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:13 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.626489+0000 mon.a (mon.0) 626 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-10-09T04:29:13.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.013 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:14.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:14.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:14.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:14.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:14.406 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:14.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.564 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:14.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.393154+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v392: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 46198/277680 objects degraded (16.637%) 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.627919+0000 osd.0 (osd.0) 1 : cluster 0 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 119'8605 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.628508+0000 mon.a (mon.0) 627 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.649552+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.743248+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.745230+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:13.746184+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: audit 2023-10-09T04:29:14.264978+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: audit 2023-10-09T04:29:14.266306+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:14.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:14 smithi153 bash[59565]: cluster 2023-10-09T04:29:14.433133+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 46198/277680 objects degraded (16.637%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:14.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:14.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:14.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:29:14.961 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.393154+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v392: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 46198/277680 objects degraded (16.637%) 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.627919+0000 osd.0 (osd.0) 1 : cluster 0 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 119'8605 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.628508+0000 mon.a (mon.0) 627 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.649552+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.743248+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.745230+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:13.746184+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: audit 2023-10-09T04:29:14.264978+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:14.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: audit 2023-10-09T04:29:14.266306+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[77063]: cluster 2023-10-09T04:29:14.433133+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 46198/277680 objects degraded (16.637%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.393154+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v392: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 46198/277680 objects degraded (16.637%) 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.627919+0000 osd.0 (osd.0) 1 : cluster 0 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 119'8605 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.628508+0000 mon.a (mon.0) 627 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.649552+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.743248+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.745230+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:13.746184+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T04:29:14.964 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: audit 2023-10-09T04:29:14.264978+0000 mon.a (mon.0) 628 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:14.964 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: audit 2023-10-09T04:29:14.266306+0000 mon.a (mon.0) 629 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:14.964 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:14 smithi082 bash[79733]: cluster 2023-10-09T04:29:14.433133+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 46198/277680 objects degraded (16.637%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:14.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:14.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:15.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:15.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:15.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:15.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:15.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:15.386 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:15.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:15.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:15.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:15.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:15.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:15.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:15.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:15.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:15.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:16.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:16.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:16.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:16.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:16.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:16.312 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:16.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:16.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:16.551 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:16.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:16.642 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:16 smithi082 bash[77063]: cluster 2023-10-09T04:29:15.394030+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v394: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 45630/274320 objects degraded (16.634%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:16.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:16.735 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:16.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:20148 5.8 5:1e8f0d6c:::benchmark_data_smithi082_106363_object20147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.749 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.852718+0000", 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 112.89266841600001, 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12782410299999999, 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.750 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 20148 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852718+0000", 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.751 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852718+0000", 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.752 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852718+0000", 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852742+0000", 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4938000000000001e-05 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.753 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852753+0000", 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0803e-05 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852761+0000", 2023-10-09T04:29:16.754 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1589999999999993e-06 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852796+0000", 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4238e-05 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.755 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.852896+0000", 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000100373 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.853155+0000", 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00025914299999999998 2023-10-09T04:29:16.756 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.854297+0000", 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00114152 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.757 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.855314+0000", 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0010169319999999999 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980454+0000", 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12513994000000001 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.758 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980499+0000", 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5649999999999998e-05 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980543+0000", 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3387999999999999e-05 2023-10-09T04:29:16.759 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:20158 5.4 e110/106 5:26a77d84:::benchmark_data_smithi082_106363_object20157:head v 110'1270, mlcod=110'1268)", 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.859074+0000", 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 112.886313065, 2023-10-09T04:29:16.760 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.123060054, 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.762 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859074+0000", 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859074+0000", 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.763 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859073+0000", 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859147+0000", 2023-10-09T04:29:16.764 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4312000000000003e-05 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859159+0000", 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1884e-05 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.765 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859167+0000", 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2819999999999999e-06 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980081+0000", 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.120914249 2023-10-09T04:29:16.766 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980118+0000", 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6866000000000002e-05 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.767 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982097+0000", 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019793559999999998 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982134+0000", 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6779000000000001e-05 2023-10-09T04:29:16.768 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:20162 5.4 e110/106 5:27f1fb29:::benchmark_data_smithi082_106363_object20161:head v 110'1271, mlcod=110'1268)", 2023-10-09T04:29:16.769 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:23.859177+0000", 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 112.88621016099999, 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12297601900000001, 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.770 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859177+0000", 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.771 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859177+0000", 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859176+0000", 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.772 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.859212+0000", 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5525999999999998e-05 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.773 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980149+0000", 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.120937008 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.980161+0000", 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.26e-05 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.774 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981023+0000", 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00086107700000000002 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.981043+0000", 2023-10-09T04:29:16.775 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0188999999999999e-05 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982137+0000", 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001093768 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.776 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:23.982153+0000", 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6263999999999999e-05 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:36852 5.b 5:d6537c2e:::benchmark_data_smithi082_106363_object36851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.462688+0000", 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 103.282699347, 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.13000811300000001, 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.777 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 36852 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462688+0000", 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.778 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462688+0000", 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462686+0000", 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.779 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462711+0000", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4544000000000001e-05 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462728+0000", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7263000000000001e-05 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.462737+0000", 2023-10-09T04:29:16.780 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4890000000000001e-06 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.465018+0000", 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0022806530000000001 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.465546+0000", 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000528066 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.781 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.465694+0000", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014830799999999999 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.467404+0000", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001709599 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469857+0000", 2023-10-09T04:29:16.782 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002452554 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479095+0000", 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00923824 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.481178+0000", 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0020834709999999999 2023-10-09T04:29:16.783 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.592696+0000", 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.111517343 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:36864 5.b 5:d6cfff89:::benchmark_data_smithi082_106363_object36863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.469817+0000", 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 103.27557015799999, 2023-10-09T04:29:16.784 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.130516193, 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 36864 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469817+0000", 2023-10-09T04:29:16.785 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469817+0000", 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469816+0000", 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.786 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469835+0000", 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9123999999999999e-05 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469842+0000", 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3119999999999999e-06 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.787 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469845+0000", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7050000000000001e-06 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.469877+0000", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1999999999999999e-05 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.470011+0000", 2023-10-09T04:29:16.788 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013355299999999999 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.470153+0000", 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000142058 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.592809+0000", 2023-10-09T04:29:16.789 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.122656011 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.598510+0000", 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0057007200000000003 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.600285+0000", 2023-10-09T04:29:16.790 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0017755570000000001 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.600311+0000", 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5964000000000001e-05 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.600333+0000", 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1611999999999999e-05 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.791 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:36867 5.b 5:dab0bb1a:::benchmark_data_smithi082_106363_object36866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.472288+0000", 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 103.273098639, 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12967246099999999, 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.792 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 36867 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472288+0000", 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.793 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472288+0000", 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472287+0000", 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.794 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472314+0000", 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7396999999999999e-05 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472328+0000", 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3754000000000001e-05 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.795 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.472337+0000", 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8580000000000006e-06 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.592866+0000", 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.120529513 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.796 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.593176+0000", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00030938999999999999 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.593567+0000", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00039114500000000002 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.598570+0000", 2023-10-09T04:29:16.797 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0050026849999999998 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.600339+0000", 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0017694270000000001 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601872+0000", 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0015332049999999999 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.798 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601921+0000", 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8643999999999997e-05 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601961+0000", 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9780000000000002e-05 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.799 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:36880 5.6 5:6cfd9a0d:::benchmark_data_smithi082_106363_object36879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.479193+0000", 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 103.266193602, 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.122123009, 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.800 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 36880 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479193+0000", 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479193+0000", 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.801 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479193+0000", 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479211+0000", 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7609999999999999e-05 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.802 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479215+0000", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479219+0000", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4309999999999998e-06 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.480917+0000", 2023-10-09T04:29:16.803 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0016979549999999999 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.481043+0000", 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012609700000000001 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.592604+0000", 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.1115611 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.804 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.597971+0000", 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053666520000000004 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.598809+0000", 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00083812399999999999 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.805 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601215+0000", 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0024063000000000001 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601274+0000", 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8566999999999997e-05 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601316+0000", 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2710999999999998e-05 2023-10-09T04:29:16.806 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:36882 5.b 5:de6e072b:::benchmark_data_smithi082_106363_object36881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:33.479644+0000", 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 103.265743359, 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.122451828, 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.807 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 36882 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479644+0000", 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.808 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479644+0000", 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479643+0000", 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479661+0000", 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8046e-05 2023-10-09T04:29:16.809 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479669+0000", 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6410000000000002e-06 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.479673+0000", 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7740000000000004e-06 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.810 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.596220+0000", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.116546565 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.596429+0000", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020947399999999999 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.596684+0000", 2023-10-09T04:29:16.811 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00025492700000000001 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601682+0000", 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049973140000000001 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.601718+0000", 2023-10-09T04:29:16.812 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6442000000000001e-05 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.602011+0000", 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00029289499999999999 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.602056+0000", 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4811999999999997e-05 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:33.602095+0000", 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9598999999999998e-05 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38611 5.6 5:62fda624:::benchmark_data_smithi082_106363_object38610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.800297+0000", 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.945089713, 2023-10-09T04:29:16.814 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.180319915, 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38611 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.815 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800297+0000", 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800297+0000", 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800296+0000", 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800334+0000", 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8241000000000002e-05 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800346+0000", 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1892000000000001e-05 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800354+0000", 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6309999999999992e-06 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800415+0000", 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.029e-05 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800558+0000", 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014312 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800659+0000", 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010146500000000001 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.802129+0000", 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0014694440000000001 2023-10-09T04:29:16.819 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.804125+0000", 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019961860000000001 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980485+0000", 2023-10-09T04:29:16.820 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17635973899999999 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980557+0000", 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2591999999999999e-05 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980617+0000", 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9833999999999997e-05 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.821 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38615 5.d 5:b9140f10:::benchmark_data_smithi082_106363_object38614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.800370+0000", 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.945016966, 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.18046802300000001, 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.822 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38615 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800370+0000", 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.823 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800370+0000", 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800369+0000", 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800400+0000", 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0979000000000003e-05 2023-10-09T04:29:16.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800407+0000", 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3050000000000004e-06 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800416+0000", 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7709999999999995e-06 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800477+0000", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0776999999999998e-05 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800605+0000", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000128255 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800756+0000", 2023-10-09T04:29:16.826 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015072299999999999 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.804199+0000", 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034427569999999998 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.804316+0000", 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000117297 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.827 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980762+0000", 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.176445939 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980803+0000", 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0858000000000001e-05 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980838+0000", 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5203000000000002e-05 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.829 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38616 5.8 5:13d14509:::benchmark_data_smithi082_106363_object38615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.800431+0000", 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.944955712, 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.18068351099999999, 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38616 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.830 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800431+0000", 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.831 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800431+0000", 2023-10-09T04:29:16.832 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.832 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.832 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.832 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800431+0000", 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800455+0000", 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4063999999999999e-05 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800471+0000", 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5934e-05 2023-10-09T04:29:16.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800476+0000", 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3720000000000004e-06 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800520+0000", 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3930000000000001e-05 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800658+0000", 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013756099999999999 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.800786+0000", 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012839700000000001 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.802434+0000", 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0016483380000000001 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980998+0000", 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17856402499999999 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981030+0000", 2023-10-09T04:29:16.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1417000000000001e-05 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981079+0000", 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9687e-05 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981115+0000", 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5336999999999997e-05 2023-10-09T04:29:16.837 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38617 5.8 5:1c9e4f30:::benchmark_data_smithi082_106363_object38616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.803167+0000", 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.94221974, 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.184728791, 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.838 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38617 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803167+0000", 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.839 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803167+0000", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803166+0000", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803192+0000", 2023-10-09T04:29:16.840 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6098999999999999e-05 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803203+0000", 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0806e-05 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803210+0000", 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.238e-06 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803252+0000", 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2030000000000002e-05 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803383+0000", 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013149 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.803530+0000", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014640599999999999 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981152+0000", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17762250299999999 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981181+0000", 2023-10-09T04:29:16.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.919e-05 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.987793+0000", 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0066117850000000002 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.987852+0000", 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8455999999999997e-05 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.987896+0000", 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4237999999999999e-05 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:38620 5.a 5:59f50c28:::benchmark_data_smithi082_106363_object38619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.806064+0000", 2023-10-09T04:29:16.845 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.939323109, 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17723397299999999, 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38620 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806064+0000", 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806064+0000", 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806063+0000", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806088+0000", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5517999999999999e-05 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806099+0000", 2023-10-09T04:29:16.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0513000000000001e-05 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806105+0000", 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9190000000000004e-06 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806145+0000", 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0475000000000001e-05 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806229+0000", 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4071000000000002e-05 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.806315+0000", 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5438000000000004e-05 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.982708+0000", 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17639317800000001 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.982897+0000", 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018924100000000001 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983102+0000", 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020450399999999999 2023-10-09T04:29:16.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983154+0000", 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2729999999999998e-05 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983298+0000", 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014342599999999999 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38616 5.8 e110/106 ondisk, result = 0)", 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.807637+0000", 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.93775030400001, 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17336452199999999, 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807637+0000", 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807637+0000", 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807636+0000", 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:29:16.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.807642+0000", 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0190000000000003e-06 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980289+0000", 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.172647052 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980304+0000", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4932999999999999e-05 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980984+0000", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00068062400000000001 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980992+0000", 2023-10-09T04:29:16.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0660000000000004e-06 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.981001+0000", 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6349999999999995e-06 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38615 5.d e110/106 ondisk, result = 0)", 2023-10-09T04:29:16.857 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.808420+0000", 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.936967088, 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17242405399999999, 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808420+0000", 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808420+0000", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808419+0000", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808425+0000", 2023-10-09T04:29:16.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8100000000000003e-06 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980325+0000", 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.171899897 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980338+0000", 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3457e-05 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980747+0000", 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040868399999999998 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980755+0000", 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8669999999999994e-06 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980844+0000", 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9029000000000004e-05 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.34794.0:38620 5.a e110/106 ondisk, result = 0)", 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:34.808789+0000", 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 101.93659741899999, 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17451841900000001, 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.862 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808789+0000", 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808789+0000", 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808789+0000", 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.808793+0000", 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5619999999999997e-06 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980692+0000", 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17189824000000001 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.980699+0000", 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4599999999999997e-06 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983087+0000", 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0023880949999999998 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983096+0000", 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5609999999999999e-06 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:34.983308+0000", 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021216400000000001 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.866 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:83133 5.b 5:d8f76180:::benchmark_data_smithi082_106363_object83132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:12.040968+0000", 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 64.704418970999996, 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.124213218, 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 83133 2023-10-09T04:29:16.867 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.040968+0000", 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.040968+0000", 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.868 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.040967+0000", 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041000+0000", 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3210999999999998e-05 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041013+0000", 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3390999999999999e-05 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041023+0000", 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8870000000000004e-06 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041140+0000", 2023-10-09T04:29:16.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000116869 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041370+0000", 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00023011799999999999 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.041574+0000", 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020405799999999999 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.050480+0000", 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.008906008 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.050578+0000", 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7812000000000005e-05 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.165079+0000", 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11450141599999999 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.165136+0000", 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7077999999999998e-05 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.165181+0000", 2023-10-09T04:29:16.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4731000000000003e-05 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:83154 5.d 5:b0cebb8a:::benchmark_data_smithi082_106363_object83153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:12.065089+0000", 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 64.680297600000003, 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12538196600000001, 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.874 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 83154 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065089+0000", 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.875 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065089+0000", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065088+0000", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065127+0000", 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9727000000000001e-05 2023-10-09T04:29:16.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065141+0000", 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3448999999999999e-05 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065150+0000", 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6799999999999999e-06 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.065203+0000", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3637999999999997e-05 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.165020+0000", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.099816480999999999 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.166308+0000", 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0012882620000000001 2023-10-09T04:29:16.878 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.173055+0000", 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0067469569999999996 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.183489+0000", 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010433852 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.190369+0000", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068801549999999998 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.190425+0000", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6317999999999998e-05 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:12.190471+0000", 2023-10-09T04:29:16.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6037e-05 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:87247 5.6 5:6c2d404c:::benchmark_data_smithi082_106363_object87246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:16.284588+0000", 2023-10-09T04:29:16.881 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 60.460799084999998, 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12186103099999999, 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87247 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.882 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284588+0000", 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284588+0000", 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284586+0000", 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284608+0000", 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1373999999999999e-05 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284618+0000", 2023-10-09T04:29:16.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8479999999999996e-06 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284624+0000", 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8100000000000003e-06 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284668+0000", 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4592999999999998e-05 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284805+0000", 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013658499999999999 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.284911+0000", 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000106816 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.886 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287647+0000", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027355349999999999 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.291306+0000", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0036585849999999998 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.406385+0000", 2023-10-09T04:29:16.887 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11507948 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.406420+0000", 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.519e-05 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.406449+0000", 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8568000000000002e-05 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.888 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:87248 5.6 5:65ac0a36:::benchmark_data_smithi082_106363_object87247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:16.287914+0000", 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 60.457472957999997, 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12874386400000001, 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:29:16.889 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87248 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287914+0000", 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287914+0000", 2023-10-09T04:29:16.890 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287807+0000", 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9998932 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287945+0000", 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000137155 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.891 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287959+0000", 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.455e-05 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.287967+0000", 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8229999999999995e-06 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.892 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.288010+0000", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3130999999999998e-05 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.288112+0000", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000101557 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.288212+0000", 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000100092 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.293147+0000", 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049355579999999996 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.296148+0000", 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0030010660000000002 2023-10-09T04:29:16.894 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.416514+0000", 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12036588500000001 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.416593+0000", 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8967000000000005e-05 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:29:16.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:16.416658+0000", 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4671999999999993e-05 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:29:16.896 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:29:16.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:16.906 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:16 smithi082 bash[79733]: cluster 2023-10-09T04:29:15.394030+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v394: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 45630/274320 objects degraded (16.634%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:16.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:16 smithi153 bash[59565]: cluster 2023-10-09T04:29:15.394030+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v394: 89 pgs: 1 unknown, 4 peering, 15 active+undersized+degraded, 14 active+undersized, 55 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 45630/274320 objects degraded (16.634%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:16.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:16.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:16.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:17.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:17.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:17.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:17.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:17.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:17.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:17.667 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:29:17.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:17.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:17.857 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:17 smithi082 bash[77063]: audit 2023-10-09T04:29:16.998568+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.82:0/1227601321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.857 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:17 smithi082 bash[77063]: audit 2023-10-09T04:29:16.999632+0000 mon.a (mon.0) 631 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.858 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:17 smithi082 bash[79733]: audit 2023-10-09T04:29:16.998568+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.82:0/1227601321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.858 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:17 smithi082 bash[79733]: audit 2023-10-09T04:29:16.999632+0000 mon.a (mon.0) 631 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:17 smithi153 bash[59565]: audit 2023-10-09T04:29:16.998568+0000 mon.c (mon.2) 49 : audit 1 from='client.? 172.21.15.82:0/1227601321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:17 smithi153 bash[59565]: audit 2023-10-09T04:29:16.999632+0000 mon.a (mon.0) 631 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:17.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:17.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:18.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:18.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:18.236 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:18.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:18.299 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:29:18.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:18.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:18.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:18.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:18.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:18.664 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:18.770 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:18.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:18.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.394719+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v395: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 50756/270921 objects degraded (18.735%); 11133/270921 objects misplaced (4.109%); 992 KiB/s, 15 objects/s recovering 2023-10-09T04:29:18.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.640985+0000 mon.a (mon.0) 632 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.641620+0000 mon.a (mon.0) 633 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.652014+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: audit 2023-10-09T04:29:17.658729+0000 mon.a (mon.0) 634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.658864+0000 mon.a (mon.0) 635 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.667527+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:18 smithi153 bash[59565]: cluster 2023-10-09T04:29:17.667656+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.926 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.394719+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v395: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 50756/270921 objects degraded (18.735%); 11133/270921 objects misplaced (4.109%); 992 KiB/s, 15 objects/s recovering 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.640985+0000 mon.a (mon.0) 632 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.641620+0000 mon.a (mon.0) 633 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.652014+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: audit 2023-10-09T04:29:17.658729+0000 mon.a (mon.0) 634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.658864+0000 mon.a (mon.0) 635 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.667527+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[79733]: cluster 2023-10-09T04:29:17.667656+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.394719+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v395: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 50756/270921 objects degraded (18.735%); 11133/270921 objects misplaced (4.109%); 992 KiB/s, 15 objects/s recovering 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.640985+0000 mon.a (mon.0) 632 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:18.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.641620+0000 mon.a (mon.0) 633 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:18.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.652014+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: audit 2023-10-09T04:29:17.658729+0000 mon.a (mon.0) 634 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:18.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.658864+0000 mon.a (mon.0) 635 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-09T04:29:18.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.667527+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:18 smithi082 bash[77063]: cluster 2023-10-09T04:29:17.667656+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T04:29:18.950 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:18.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:19.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:19.080 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:19.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:19.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:19.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:19.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:19.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:19.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:19.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:19.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:19.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:19.738 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:19 smithi153 bash[59565]: cluster 2023-10-09T04:29:19.434307+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 50756/270921 objects degraded (18.735%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:19.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:19.839 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:29:19.839 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:19.840 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T04:29:19.840 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:29:19.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:19.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:19.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:19.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:19 smithi082 bash[77063]: cluster 2023-10-09T04:29:19.434307+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 50756/270921 objects degraded (18.735%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:19.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:19 smithi082 bash[79733]: cluster 2023-10-09T04:29:19.434307+0000 mon.a (mon.0) 636 : cluster 3 Health check update: Degraded data redundancy: 50756/270921 objects degraded (18.735%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:19] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-09T04:29:19.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:20.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:20.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:20.267 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:20.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:20.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:20.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:20.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:20.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:20.668 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:20 smithi082 bash[79733]: cluster 2023-10-09T04:29:19.395453+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v397: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 50549/268851 objects degraded (18.802%); 11133/268851 objects misplaced (4.141%); 878 KiB/s, 13 objects/s recovering 2023-10-09T04:29:20.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:20.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:20.817 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:20.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:20.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:20 smithi153 bash[59565]: cluster 2023-10-09T04:29:19.395453+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v397: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 50549/268851 objects degraded (18.802%); 11133/268851 objects misplaced (4.141%); 878 KiB/s, 13 objects/s recovering 2023-10-09T04:29:20.949 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:20.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:20.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:20 smithi082 bash[77063]: cluster 2023-10-09T04:29:19.395453+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v397: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 50549/268851 objects degraded (18.802%); 11133/268851 objects misplaced (4.141%); 878 KiB/s, 13 objects/s recovering 2023-10-09T04:29:21.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:21.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:21.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:29:21.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:21.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:21.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:21.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:21.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:21.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:21.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:21.610 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:21.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:21.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:21.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:21.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:21.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:21.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:22.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:22.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:22.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:22.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:22.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.668 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:22.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:22.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:22 smithi153 bash[59565]: cluster 2023-10-09T04:29:21.396068+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v398: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 49845/265245 objects degraded (18.792%); 11133/265245 objects misplaced (4.197%); 744 KiB/s, 11 objects/s recovering 2023-10-09T04:29:22.818 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:22.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:22.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:22 smithi082 bash[77063]: cluster 2023-10-09T04:29:21.396068+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v398: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 49845/265245 objects degraded (18.792%); 11133/265245 objects misplaced (4.197%); 744 KiB/s, 11 objects/s recovering 2023-10-09T04:29:22.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:22 smithi082 bash[79733]: cluster 2023-10-09T04:29:21.396068+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v398: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 49845/265245 objects degraded (18.792%); 11133/265245 objects misplaced (4.197%); 744 KiB/s, 11 objects/s recovering 2023-10-09T04:29:23.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:23.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:23.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:23.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:23.223 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:23.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:23.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:23.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:23.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:23.606 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:23.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:23.682 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:23.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:23.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:23.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:23.901 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:23.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:24.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:24.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:24.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:24.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:24.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:24.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:24.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:24.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:24.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:24.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:24.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:24.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:24.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:24.704 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:24.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:24.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:24.881 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:24.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:24.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:24 smithi153 bash[59565]: cluster 2023-10-09T04:29:23.396763+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v399: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 48860/259491 objects degraded (18.829%); 10617/259491 objects misplaced (4.091%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:29:24.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:24 smithi153 bash[59565]: cluster 2023-10-09T04:29:24.436948+0000 mon.a (mon.0) 637 : cluster 3 Health check update: Degraded data redundancy: 48860/259491 objects degraded (18.829%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:24.955 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:24 smithi082 bash[77063]: cluster 2023-10-09T04:29:23.396763+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v399: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 48860/259491 objects degraded (18.829%); 10617/259491 objects misplaced (4.091%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:29:24.956 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:24 smithi082 bash[77063]: cluster 2023-10-09T04:29:24.436948+0000 mon.a (mon.0) 637 : cluster 3 Health check update: Degraded data redundancy: 48860/259491 objects degraded (18.829%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:24.956 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:24 smithi082 bash[79733]: cluster 2023-10-09T04:29:23.396763+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v399: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 48860/259491 objects degraded (18.829%); 10617/259491 objects misplaced (4.091%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:29:24.956 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:24 smithi082 bash[79733]: cluster 2023-10-09T04:29:24.436948+0000 mon.a (mon.0) 637 : cluster 3 Health check update: Degraded data redundancy: 48860/259491 objects degraded (18.829%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:24.987 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.278 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:25.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:25.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.721 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:25.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:25.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:25.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:25.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:26.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:26.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:26.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:26.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:26.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:26.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:26.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:26.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:26.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:26.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:26 smithi082 bash[77063]: cluster 2023-10-09T04:29:25.397556+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v400: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 48000/254157 objects degraded (18.886%); 10617/254157 objects misplaced (4.177%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:29:26.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:26 smithi082 bash[79733]: cluster 2023-10-09T04:29:25.397556+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v400: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 48000/254157 objects degraded (18.886%); 10617/254157 objects misplaced (4.177%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:29:26.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:26.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:26.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:26.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:26.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:26 smithi153 bash[59565]: cluster 2023-10-09T04:29:25.397556+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v400: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 48000/254157 objects degraded (18.886%); 10617/254157 objects misplaced (4.177%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:29:26.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:26.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:26.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:26.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:27.084 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:27.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:27.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:27.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:27.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:27.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:27.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:27.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:27.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:27.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:27.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:27.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:27.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:27.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:27.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:28.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:28.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:28.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:28.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:28.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:28.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:28.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:28.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:28.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:28 smithi082 bash[77063]: cluster 2023-10-09T04:29:27.398376+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v401: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 47037/248556 objects degraded (18.924%); 10130/248556 objects misplaced (4.076%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:29:28.718 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:28 smithi082 bash[79733]: cluster 2023-10-09T04:29:27.398376+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v401: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 47037/248556 objects degraded (18.924%); 10130/248556 objects misplaced (4.076%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:29:28.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:28.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.865 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:28.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.987 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:28.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:29.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:29.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:28 smithi153 bash[59565]: cluster 2023-10-09T04:29:27.398376+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v401: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 47037/248556 objects degraded (18.924%); 10130/248556 objects misplaced (4.076%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:29:29.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:29.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:29.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:29.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:29.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:29.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:29.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:29.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:29.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:29.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.907 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:29] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:29:29.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:29.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:30.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:30.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:30.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:30.195 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:30.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:30.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:30 smithi153 bash[59565]: audit 2023-10-09T04:29:29.262811+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:30 smithi153 bash[59565]: audit 2023-10-09T04:29:29.267492+0000 mon.a (mon.0) 639 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:30 smithi153 bash[59565]: cluster 2023-10-09T04:29:29.398990+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v402: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 46815/246426 objects degraded (18.998%); 10130/246426 objects misplaced (4.111%); 708 KiB/s, 11 objects/s recovering 2023-10-09T04:29:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:30 smithi153 bash[59565]: cluster 2023-10-09T04:29:29.438025+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 47037/248556 objects degraded (18.924%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:30.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:30.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:30.566 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[77063]: audit 2023-10-09T04:29:29.262811+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[77063]: audit 2023-10-09T04:29:29.267492+0000 mon.a (mon.0) 639 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[77063]: cluster 2023-10-09T04:29:29.398990+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v402: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 46815/246426 objects degraded (18.998%); 10130/246426 objects misplaced (4.111%); 708 KiB/s, 11 objects/s recovering 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[77063]: cluster 2023-10-09T04:29:29.438025+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 47037/248556 objects degraded (18.924%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[79733]: audit 2023-10-09T04:29:29.262811+0000 mon.a (mon.0) 638 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[79733]: audit 2023-10-09T04:29:29.267492+0000 mon.a (mon.0) 639 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:30.574 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[79733]: cluster 2023-10-09T04:29:29.398990+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v402: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 46815/246426 objects degraded (18.998%); 10130/246426 objects misplaced (4.111%); 708 KiB/s, 11 objects/s recovering 2023-10-09T04:29:30.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:30 smithi082 bash[79733]: cluster 2023-10-09T04:29:29.438025+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 47037/248556 objects degraded (18.924%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:29:30.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:30.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:30.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:30.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:30.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:30.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:30.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:30.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:31.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:31.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:31.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:31.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.370 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:31.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:31.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:31.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:31.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:31.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:31.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:31.872 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.914 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:31.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:32.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:32.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:32.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:32.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:32.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:32.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:32.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:32.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:32.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:32 smithi153 bash[59565]: cluster 2023-10-09T04:29:31.399845+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v403: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 46130/242988 objects degraded (18.984%); 10130/242988 objects misplaced (4.169%); 693 KiB/s, 10 objects/s recovering 2023-10-09T04:29:32.686 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.694 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:32 smithi082 bash[77063]: cluster 2023-10-09T04:29:31.399845+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v403: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 46130/242988 objects degraded (18.984%); 10130/242988 objects misplaced (4.169%); 693 KiB/s, 10 objects/s recovering 2023-10-09T04:29:32.697 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:32 smithi082 bash[79733]: cluster 2023-10-09T04:29:31.399845+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v403: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 46130/242988 objects degraded (18.984%); 10130/242988 objects misplaced (4.169%); 693 KiB/s, 10 objects/s recovering 2023-10-09T04:29:32.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:32.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:32.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:32.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:33.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:33.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:33.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:33.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:33.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.386 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:33.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:33.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.665 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:33.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:33.790 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:33.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:33.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:33.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:34.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:34.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:34.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:34.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:34.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:34.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:34.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:34.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:34 smithi082 bash[77063]: cluster 2023-10-09T04:29:33.404760+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v404: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 45094/237456 objects degraded (18.990%); 9659/237456 objects misplaced (4.068%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:29:34.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:34 smithi082 bash[77063]: cluster 2023-10-09T04:29:34.439071+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 45094/237456 objects degraded (18.990%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:34.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:34 smithi082 bash[79733]: cluster 2023-10-09T04:29:33.404760+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v404: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 45094/237456 objects degraded (18.990%); 9659/237456 objects misplaced (4.068%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:29:34.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:34 smithi082 bash[79733]: cluster 2023-10-09T04:29:34.439071+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 45094/237456 objects degraded (18.990%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:34.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:34.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:34 smithi153 bash[59565]: cluster 2023-10-09T04:29:33.404760+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v404: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 45094/237456 objects degraded (18.990%); 9659/237456 objects misplaced (4.068%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:29:34.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:34 smithi153 bash[59565]: cluster 2023-10-09T04:29:34.439071+0000 mon.a (mon.0) 641 : cluster 3 Health check update: Degraded data redundancy: 45094/237456 objects degraded (18.990%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:34.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:34.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:34.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:34.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:34.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:34.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:35.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:35.099 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:35.203 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:35.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:35.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:35.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:35.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:35.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:35.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:35.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:35.632 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:35.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:35.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:35.769 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:35.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:35.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:35.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:36.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:36.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:36.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:36.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:36.470 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:36 smithi082 bash[77063]: cluster 2023-10-09T04:29:35.405700+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v405: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 44071/231549 objects degraded (19.033%); 9659/231549 objects misplaced (4.171%); 799 KiB/s, 12 objects/s recovering 2023-10-09T04:29:36.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:36 smithi153 bash[59565]: cluster 2023-10-09T04:29:35.405700+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v405: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 44071/231549 objects degraded (19.033%); 9659/231549 objects misplaced (4.171%); 799 KiB/s, 12 objects/s recovering 2023-10-09T04:29:36.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.655 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:36 smithi082 bash[79733]: cluster 2023-10-09T04:29:35.405700+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v405: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 44071/231549 objects degraded (19.033%); 9659/231549 objects misplaced (4.171%); 799 KiB/s, 12 objects/s recovering 2023-10-09T04:29:36.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:36.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:36.930 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:36.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:36.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:37.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:37.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:37.180 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:29:37.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:37.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:37.241 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:29:37.282 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:29:37.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:37.326 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:29:37.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:37.359 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:29:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:37.415 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:29:37.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:37.465 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.467 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:37.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:37.588 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:37.588 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:37.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:37.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:37 smithi153 bash[59565]: audit 2023-10-09T04:29:37.139606+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.82:0/2090746116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:37.619 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:37 smithi082 bash[77063]: audit 2023-10-09T04:29:37.139606+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.82:0/2090746116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:37.620 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:37 smithi082 bash[79733]: audit 2023-10-09T04:29:37.139606+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.82:0/2090746116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:37.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:37.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:37.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:37.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:37.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:37.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:38.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:38.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:38.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:38.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:38.316 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:29:38.327 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:29:38.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:38.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:38.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:38.515 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:29:38.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:38.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[77063]: cluster 2023-10-09T04:29:37.406520+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v406: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 42975/225816 objects degraded (19.031%); 9228/225816 objects misplaced (4.087%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[77063]: audit 2023-10-09T04:29:38.396267+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.82:0/1483054851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[77063]: audit 2023-10-09T04:29:38.397378+0000 mon.a (mon.0) 642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[79733]: cluster 2023-10-09T04:29:37.406520+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v406: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 42975/225816 objects degraded (19.031%); 9228/225816 objects misplaced (4.087%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[79733]: audit 2023-10-09T04:29:38.396267+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.82:0/1483054851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.611 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:38 smithi082 bash[79733]: audit 2023-10-09T04:29:38.397378+0000 mon.a (mon.0) 642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:38.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:38.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:38.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:38.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:38.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:38 smithi153 bash[59565]: cluster 2023-10-09T04:29:37.406520+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v406: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 42975/225816 objects degraded (19.031%); 9228/225816 objects misplaced (4.087%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:29:38.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:38 smithi153 bash[59565]: audit 2023-10-09T04:29:38.396267+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.82:0/1483054851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:38 smithi153 bash[59565]: audit 2023-10-09T04:29:38.397378+0000 mon.a (mon.0) 642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:39.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:39.120 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:39.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:39.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:39.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:39.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.411 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:39.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:39.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:39.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:39.561 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: audit 2023-10-09T04:29:38.289048+0000 mgr.y (mgr.34107) 458 : audit 0 from='client.44804 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:39.561 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: cluster 2023-10-09T04:29:38.488644+0000 mon.a (mon.0) 643 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.561 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: audit 2023-10-09T04:29:38.503667+0000 mon.a (mon.0) 644 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: cluster 2023-10-09T04:29:38.503893+0000 mon.a (mon.0) 645 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: cluster 2023-10-09T04:29:38.508687+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[77063]: cluster 2023-10-09T04:29:39.440263+0000 mon.a (mon.0) 646 : cluster 3 Health check update: Degraded data redundancy: 42975/225816 objects degraded (19.031%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: audit 2023-10-09T04:29:38.289048+0000 mgr.y (mgr.34107) 458 : audit 0 from='client.44804 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: cluster 2023-10-09T04:29:38.488644+0000 mon.a (mon.0) 643 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: audit 2023-10-09T04:29:38.503667+0000 mon.a (mon.0) 644 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: cluster 2023-10-09T04:29:38.503893+0000 mon.a (mon.0) 645 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: cluster 2023-10-09T04:29:38.508687+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:39.562 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[79733]: cluster 2023-10-09T04:29:39.440263+0000 mon.a (mon.0) 646 : cluster 3 Health check update: Degraded data redundancy: 42975/225816 objects degraded (19.031%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:39.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: audit 2023-10-09T04:29:38.289048+0000 mgr.y (mgr.34107) 458 : audit 0 from='client.44804 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: cluster 2023-10-09T04:29:38.488644+0000 mon.a (mon.0) 643 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: audit 2023-10-09T04:29:38.503667+0000 mon.a (mon.0) 644 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: cluster 2023-10-09T04:29:38.503893+0000 mon.a (mon.0) 645 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-09T04:29:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: cluster 2023-10-09T04:29:38.508687+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:39 smithi153 bash[59565]: cluster 2023-10-09T04:29:39.440263+0000 mon.a (mon.0) 646 : cluster 3 Health check update: Degraded data redundancy: 42975/225816 objects degraded (19.031%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:39.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:39.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:39.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:39.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:39.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:39.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:39.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:39] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:29:40.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:40.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:40.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:40.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:40.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:40.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:40.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:40.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:40.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:40.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:40.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:40.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:40.711 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:40 smithi153 bash[59565]: cluster 2023-10-09T04:29:38.514512+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:40.711 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:40 smithi153 bash[59565]: cluster 2023-10-09T04:29:39.407243+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v408: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 42708/223686 objects degraded (19.093%); 9228/223686 objects misplaced (4.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:40.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:40.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:40.841 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:40 smithi082 bash[77063]: cluster 2023-10-09T04:29:38.514512+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:40.842 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:40 smithi082 bash[77063]: cluster 2023-10-09T04:29:39.407243+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v408: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 42708/223686 objects degraded (19.093%); 9228/223686 objects misplaced (4.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:40.842 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:40 smithi082 bash[79733]: cluster 2023-10-09T04:29:38.514512+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T04:29:40.842 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:40 smithi082 bash[79733]: cluster 2023-10-09T04:29:39.407243+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v408: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 42708/223686 objects degraded (19.093%); 9228/223686 objects misplaced (4.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:40.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:40.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:40.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:40.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:41.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:41.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:41.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:41.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:29:41.226 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:29:41.252 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:41.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:41.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:29:41.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:41.385 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":406,"stamp":"2023-10-09T04:29:37.405880+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4935915467,"num_objects":75272,"num_object_clones":0,"num_object_copies":225816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":42975,"num_objects_misplaced":9228,"num_objects_unfound":0,"num_objects_dirty":75272,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":154320,"num_write_kb":7325752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":397,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142484,"ondisk_log_size":142484,"up":236,"acting":236,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16959172,"kb_used_data":15381764,"kb_used_omap":34,"kb_used_meta":1577181,"kb_avail":733002044,"statfs":{"total":767960285184,"available":750594093056,"internally_reserved":0,"allocated":15750926336,"data_stored":15746553033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1615034069},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,8,5,7,4,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-619118592,"num_objects":-9447,"num_object_clones":0,"num_object_copies":-28341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-5025,"num_objects_misplaced":-1389,"num_objects_unfound":0,"num_objects_dirty":-9447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9447,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":235,"num_bytes_recovered":15400960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008921"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":105,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.654801+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:29:13.654801+0000","last_peered":"2023-10-09T04:29:13.654801+0000","last_clean":"2023-10-09T04:29:13.654801+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:29:13.654801+0000","last_undegraded":"2023-10-09T04:29:13.654801+0000","last_fullsized":"2023-10-09T04:29:13.654801+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.627961+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:29:13.627961+0000","last_peered":"2023-10-09T04:29:13.627961+0000","last_clean":"2023-10-09T04:29:13.627961+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:29:13.627961+0000","last_undegraded":"2023-10-09T04:29:13.627961+0000","last_fullsized":"2023-10-09T04:29:13.627961+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677786+0000","last_change":"2023-10-09T04:29:12.641467+0000","last_active":"2023-10-09T04:29:13.677786+0000","last_peered":"2023-10-09T04:29:13.677786+0000","last_clean":"2023-10-09T04:29:13.677786+0000","last_became_active":"2023-10-09T04:29:12.641036+0000","last_became_peered":"2023-10-09T04:29:12.641036+0000","last_unstale":"2023-10-09T04:29:13.677786+0000","last_undegraded":"2023-10-09T04:29:13.677786+0000","last_fullsized":"2023-10-09T04:29:13.677786+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:41.624520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":74,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.627626+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:29:13.627626+0000","last_peered":"2023-10-09T04:29:13.627626+0000","last_clean":"2023-10-09T04:29:13.627626+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:29:13.627626+0000","last_undegraded":"2023-10-09T04:29:13.627626+0000","last_fullsized":"2023-10-09T04:29:13.627626+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":97,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.625193+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:29:13.625193+0000","last_peered":"2023-10-09T04:29:13.625193+0000","last_clean":"2023-10-09T04:29:13.625193+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:29:13.625193+0000","last_undegraded":"2023-10-09T04:29:13.625193+0000","last_fullsized":"2023-10-09T04:29:13.625193+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.751661+0000","last_change":"2023-10-09T04:28:27.618032+0000","last_active":"2023-10-09T04:29:13.751661+0000","last_peered":"2023-10-09T04:29:13.751661+0000","last_clean":"2023-10-09T04:28:19.577947+0000","last_became_active":"2023-10-09T04:28:27.618032+0000","last_became_peered":"2023-10-09T04:28:27.618032+0000","last_unstale":"2023-10-09T04:29:13.751661+0000","last_undegraded":"2023-10-09T04:29:13.751661+0000","last_fullsized":"2023-10-09T04:28:27.603354+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":80,"reported_epoch":123,"state":"active+undersized","last_fresh":"2023-10-09T04:29:17.672197+0000","last_change":"2023-10-09T04:28:27.599874+0000","last_active":"2023-10-09T04:29:17.672197+0000","last_peered":"2023-10-09T04:29:17.672197+0000","last_clean":"2023-10-09T04:28:19.571527+0000","last_became_active":"2023-10-09T04:28:27.599874+0000","last_became_peered":"2023-10-09T04:28:27.599874+0000","last_unstale":"2023-10-09T04:29:17.672197+0000","last_undegraded":"2023-10-09T04:29:17.672197+0000","last_fullsized":"2023-10-09T04:28:27.597863+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:45.767094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":81,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677512+0000","last_change":"2023-10-09T04:28:51.528372+0000","last_active":"2023-10-09T04:29:13.677512+0000","last_peered":"2023-10-09T04:29:13.677512+0000","last_clean":"2023-10-09T04:29:13.677512+0000","last_became_active":"2023-10-09T04:28:51.527988+0000","last_became_peered":"2023-10-09T04:28:51.527988+0000","last_unstale":"2023-10-09T04:29:13.677512+0000","last_undegraded":"2023-10-09T04:29:13.677512+0000","last_fullsized":"2023-10-09T04:29:13.677512+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.258888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.753139+0000","last_change":"2023-10-09T04:28:27.608355+0000","last_active":"2023-10-09T04:29:13.753139+0000","last_peered":"2023-10-09T04:29:13.753139+0000","last_clean":"2023-10-09T04:22:23.598572+0000","last_became_active":"2023-10-09T04:28:27.608355+0000","last_became_peered":"2023-10-09T04:28:27.608355+0000","last_unstale":"2023-10-09T04:29:13.753139+0000","last_undegraded":"2023-10-09T04:29:13.753139+0000","last_fullsized":"2023-10-09T04:28:27.604613+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.752340+0000","last_change":"2023-10-09T04:28:51.533081+0000","last_active":"2023-10-09T04:29:13.752340+0000","last_peered":"2023-10-09T04:29:13.752340+0000","last_clean":"2023-10-09T04:29:13.752340+0000","last_became_active":"2023-10-09T04:28:51.532684+0000","last_became_peered":"2023-10-09T04:28:51.532684+0000","last_unstale":"2023-10-09T04:29:13.752340+0000","last_undegraded":"2023-10-09T04:29:13.752340+0000","last_fullsized":"2023-10-09T04:29:13.752340+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":92,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753172+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:29:13.753172+0000","last_peered":"2023-10-09T04:29:13.753172+0000","last_clean":"2023-10-09T04:29:13.753172+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:29:13.753172+0000","last_undegraded":"2023-10-09T04:29:13.753172+0000","last_fullsized":"2023-10-09T04:29:13.753172+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677661+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:29:13.677661+0000","last_peered":"2023-10-09T04:29:13.677661+0000","last_clean":"2023-10-09T04:29:13.677661+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:29:13.677661+0000","last_undegraded":"2023-10-09T04:29:13.677661+0000","last_fullsized":"2023-10-09T04:29:13.677661+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":96,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.655920+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:29:13.655920+0000","last_peered":"2023-10-09T04:29:13.655920+0000","last_clean":"2023-10-09T04:29:13.655920+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:29:13.655920+0000","last_undegraded":"2023-10-09T04:29:13.655920+0000","last_fullsized":"2023-10-09T04:29:13.655920+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.655038+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:29:13.655038+0000","last_peered":"2023-10-09T04:29:13.655038+0000","last_clean":"2023-10-09T04:29:13.655038+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:29:13.655038+0000","last_undegraded":"2023-10-09T04:29:13.655038+0000","last_fullsized":"2023-10-09T04:29:13.655038+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":91,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753233+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:29:13.753233+0000","last_peered":"2023-10-09T04:29:13.753233+0000","last_clean":"2023-10-09T04:29:13.753233+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:29:13.753233+0000","last_undegraded":"2023-10-09T04:29:13.753233+0000","last_fullsized":"2023-10-09T04:29:13.753233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.678195+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:29:13.678195+0000","last_peered":"2023-10-09T04:29:13.678195+0000","last_clean":"2023-10-09T04:29:13.678195+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:29:13.678195+0000","last_undegraded":"2023-10-09T04:29:13.678195+0000","last_fullsized":"2023-10-09T04:29:13.678195+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":74,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:17.672071+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:29:17.672071+0000","last_peered":"2023-10-09T04:29:17.672071+0000","last_clean":"2023-10-09T04:29:17.672071+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:29:17.672071+0000","last_undegraded":"2023-10-09T04:29:17.672071+0000","last_fullsized":"2023-10-09T04:29:17.672071+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":78,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.752785+0000","last_change":"2023-10-09T04:28:27.609484+0000","last_active":"2023-10-09T04:29:13.752785+0000","last_peered":"2023-10-09T04:29:13.752785+0000","last_clean":"2023-10-09T04:23:18.758860+0000","last_became_active":"2023-10-09T04:28:27.609484+0000","last_became_peered":"2023-10-09T04:28:27.609484+0000","last_unstale":"2023-10-09T04:29:13.752785+0000","last_undegraded":"2023-10-09T04:29:13.752785+0000","last_fullsized":"2023-10-09T04:28:27.606325+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.752742+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:29:13.752742+0000","last_peered":"2023-10-09T04:29:13.752742+0000","last_clean":"2023-10-09T04:29:13.752742+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:29:13.752742+0000","last_undegraded":"2023-10-09T04:29:13.752742+0000","last_fullsized":"2023-10-09T04:29:13.752742+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":74,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:17.672512+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:29:17.672512+0000","last_peered":"2023-10-09T04:29:17.672512+0000","last_clean":"2023-10-09T04:29:17.672512+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:29:17.672512+0000","last_undegraded":"2023-10-09T04:29:17.672512+0000","last_fullsized":"2023-10-09T04:29:17.672512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":103,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.751008+0000","last_change":"2023-10-09T04:28:27.608819+0000","last_active":"2023-10-09T04:29:13.751008+0000","last_peered":"2023-10-09T04:29:13.751008+0000","last_clean":"2023-10-09T04:28:19.577481+0000","last_became_active":"2023-10-09T04:28:27.608819+0000","last_became_peered":"2023-10-09T04:28:27.608819+0000","last_unstale":"2023-10-09T04:29:13.751008+0000","last_undegraded":"2023-10-09T04:28:27.602582+0000","last_fullsized":"2023-10-09T04:28:27.602406+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753196+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:29:13.753196+0000","last_peered":"2023-10-09T04:29:13.753196+0000","last_clean":"2023-10-09T04:29:13.753196+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:29:13.753196+0000","last_undegraded":"2023-10-09T04:29:13.753196+0000","last_fullsized":"2023-10-09T04:29:13.753196+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"123'9682","reported_seq":9707,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:35.239565+0000","last_change":"2023-10-09T04:26:55.440461+0000","last_active":"2023-10-09T04:29:35.239565+0000","last_peered":"2023-10-09T04:29:35.239565+0000","last_clean":"2023-10-09T04:29:35.239565+0000","last_became_active":"2023-10-09T04:26:55.440099+0000","last_became_peered":"2023-10-09T04:26:55.440099+0000","last_unstale":"2023-10-09T04:29:35.239565+0000","last_undegraded":"2023-10-09T04:29:35.239565+0000","last_fullsized":"2023-10-09T04:29:35.239565+0000","mapping_epoch":106,"log_start":"110'1100","ondisk_log_start":"110'1100","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8582,"log_dups_size":0,"ondisk_log_size":8582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312999936,"num_objects":4776,"num_object_clones":0,"num_object_copies":14328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9682,"num_write_kb":462656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":90,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753317+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:29:13.753317+0000","last_peered":"2023-10-09T04:29:13.753317+0000","last_clean":"2023-10-09T04:29:13.753317+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:29:13.753317+0000","last_undegraded":"2023-10-09T04:29:13.753317+0000","last_fullsized":"2023-10-09T04:29:13.753317+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.752184+0000","last_change":"2023-10-09T04:28:27.610146+0000","last_active":"2023-10-09T04:29:13.752184+0000","last_peered":"2023-10-09T04:29:13.752184+0000","last_clean":"2023-10-09T04:11:26.190073+0000","last_became_active":"2023-10-09T04:28:27.610146+0000","last_became_peered":"2023-10-09T04:28:27.610146+0000","last_unstale":"2023-10-09T04:29:13.752184+0000","last_undegraded":"2023-10-09T04:29:13.752184+0000","last_fullsized":"2023-10-09T04:28:27.604381+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"123'9650","reported_seq":9677,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:37.096739+0000","last_change":"2023-10-09T04:26:55.433159+0000","last_active":"2023-10-09T04:29:37.096739+0000","last_peered":"2023-10-09T04:29:37.096739+0000","last_clean":"2023-10-09T04:29:37.096739+0000","last_became_active":"2023-10-09T04:26:55.432784+0000","last_became_peered":"2023-10-09T04:26:55.432784+0000","last_unstale":"2023-10-09T04:29:37.096739+0000","last_undegraded":"2023-10-09T04:29:37.096739+0000","last_fullsized":"2023-10-09T04:29:37.096739+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":9650,"log_dups_size":0,"ondisk_log_size":9650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:19.007979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303431680,"num_objects":4630,"num_object_clones":0,"num_object_copies":13890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9650,"num_write_kb":456960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":104,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753392+0000","last_change":"2023-10-09T04:29:12.722710+0000","last_active":"2023-10-09T04:29:13.753392+0000","last_peered":"2023-10-09T04:29:13.753392+0000","last_clean":"2023-10-09T04:29:13.753392+0000","last_became_active":"2023-10-09T04:29:12.635166+0000","last_became_peered":"2023-10-09T04:29:12.635166+0000","last_unstale":"2023-10-09T04:29:13.753392+0000","last_undegraded":"2023-10-09T04:29:13.753392+0000","last_fullsized":"2023-10-09T04:29:13.753392+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":103,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.655278+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:29:13.655278+0000","last_peered":"2023-10-09T04:29:13.655278+0000","last_clean":"2023-10-09T04:29:13.655278+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:29:13.655278+0000","last_undegraded":"2023-10-09T04:29:13.655278+0000","last_fullsized":"2023-10-09T04:29:13.655278+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"123'9741","reported_seq":9771,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:37.099213+0000","last_change":"2023-10-09T04:28:27.604547+0000","last_active":"2023-10-09T04:29:37.099213+0000","last_peered":"2023-10-09T04:29:37.099213+0000","last_clean":"2023-10-09T04:28:26.590593+0000","last_became_active":"2023-10-09T04:28:27.604547+0000","last_became_peered":"2023-10-09T04:28:27.604547+0000","last_unstale":"2023-10-09T04:29:37.099213+0000","last_undegraded":"2023-10-09T04:28:27.593309+0000","last_fullsized":"2023-10-09T04:28:27.593130+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":9741,"log_dups_size":0,"ondisk_log_size":9741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:32.223773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303235072,"num_objects":4627,"num_object_clones":0,"num_object_copies":13881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9741,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":4627}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":81,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.753489+0000","last_change":"2023-10-09T04:28:27.610412+0000","last_active":"2023-10-09T04:29:13.753489+0000","last_peered":"2023-10-09T04:29:13.753489+0000","last_clean":"2023-10-09T04:28:19.573266+0000","last_became_active":"2023-10-09T04:28:27.610412+0000","last_became_peered":"2023-10-09T04:28:27.610412+0000","last_unstale":"2023-10-09T04:29:13.753489+0000","last_undegraded":"2023-10-09T04:28:27.605925+0000","last_fullsized":"2023-10-09T04:28:27.605804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.751204+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:29:13.751204+0000","last_peered":"2023-10-09T04:29:13.751204+0000","last_clean":"2023-10-09T04:29:13.751204+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:29:13.751204+0000","last_undegraded":"2023-10-09T04:29:13.751204+0000","last_fullsized":"2023-10-09T04:29:13.751204+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"123'9576","reported_seq":9603,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:37.036813+0000","last_change":"2023-10-09T04:26:55.435227+0000","last_active":"2023-10-09T04:29:37.036813+0000","last_peered":"2023-10-09T04:29:37.036813+0000","last_clean":"2023-10-09T04:29:37.036813+0000","last_became_active":"2023-10-09T04:26:55.434919+0000","last_became_peered":"2023-10-09T04:26:55.434919+0000","last_unstale":"2023-10-09T04:29:37.036813+0000","last_undegraded":"2023-10-09T04:29:37.036813+0000","last_fullsized":"2023-10-09T04:29:37.036813+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":9576,"log_dups_size":0,"ondisk_log_size":9576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:38.324241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":78,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.750937+0000","last_change":"2023-10-09T04:28:51.527885+0000","last_active":"2023-10-09T04:29:13.750937+0000","last_peered":"2023-10-09T04:29:13.750937+0000","last_clean":"2023-10-09T04:23:18.758933+0000","last_became_active":"2023-10-09T04:28:51.527885+0000","last_became_peered":"2023-10-09T04:28:51.527885+0000","last_unstale":"2023-10-09T04:29:13.750937+0000","last_undegraded":"2023-10-09T04:29:13.750937+0000","last_fullsized":"2023-10-09T04:28:51.525148+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":87,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.626706+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:29:13.626706+0000","last_peered":"2023-10-09T04:29:13.626706+0000","last_clean":"2023-10-09T04:29:13.626706+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:29:13.626706+0000","last_undegraded":"2023-10-09T04:29:13.626706+0000","last_fullsized":"2023-10-09T04:29:13.626706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"123'9703","reported_seq":9736,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:37.162172+0000","last_change":"2023-10-09T04:28:27.610140+0000","last_active":"2023-10-09T04:29:37.162172+0000","last_peered":"2023-10-09T04:29:37.162172+0000","last_clean":"2023-10-09T04:28:26.568395+0000","last_became_active":"2023-10-09T04:28:27.610140+0000","last_became_peered":"2023-10-09T04:28:27.610140+0000","last_unstale":"2023-10-09T04:29:37.162172+0000","last_undegraded":"2023-10-09T04:28:27.592853+0000","last_fullsized":"2023-10-09T04:28:27.592674+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":9703,"log_dups_size":0,"ondisk_log_size":9703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:18.245131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303955992,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4639,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9703,"num_write_kb":458881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":4639}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":37,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.626831+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:29:13.626831+0000","last_peered":"2023-10-09T04:29:13.626831+0000","last_clean":"2023-10-09T04:29:13.626831+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:29:13.626831+0000","last_undegraded":"2023-10-09T04:29:13.626831+0000","last_fullsized":"2023-10-09T04:29:13.626831+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.656215+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:29:13.656215+0000","last_peered":"2023-10-09T04:29:13.656215+0000","last_clean":"2023-10-09T04:29:13.656215+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:29:13.656215+0000","last_undegraded":"2023-10-09T04:29:13.656215+0000","last_fullsized":"2023-10-09T04:29:13.656215+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"123'9524","reported_seq":9551,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:34.044610+0000","last_change":"2023-10-09T04:26:55.433498+0000","last_active":"2023-10-09T04:29:34.044610+0000","last_peered":"2023-10-09T04:29:34.044610+0000","last_clean":"2023-10-09T04:29:34.044610+0000","last_became_active":"2023-10-09T04:26:55.433162+0000","last_became_peered":"2023-10-09T04:26:55.433162+0000","last_unstale":"2023-10-09T04:29:34.044610+0000","last_undegraded":"2023-10-09T04:29:34.044610+0000","last_fullsized":"2023-10-09T04:29:34.044610+0000","mapping_epoch":106,"log_start":"110'1400","ondisk_log_start":"110'1400","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:56.313689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313655296,"num_objects":4786,"num_object_clones":0,"num_object_copies":14358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9524,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.751586+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:29:13.751586+0000","last_peered":"2023-10-09T04:29:13.751586+0000","last_clean":"2023-10-09T04:29:13.751586+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:29:13.751586+0000","last_undegraded":"2023-10-09T04:29:13.751586+0000","last_fullsized":"2023-10-09T04:29:13.751586+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.627842+0000","last_change":"2023-10-09T04:29:12.637984+0000","last_active":"2023-10-09T04:29:13.627842+0000","last_peered":"2023-10-09T04:29:13.627842+0000","last_clean":"2023-10-09T04:29:13.627842+0000","last_became_active":"2023-10-09T04:29:12.637551+0000","last_became_peered":"2023-10-09T04:29:12.637551+0000","last_unstale":"2023-10-09T04:29:13.627842+0000","last_undegraded":"2023-10-09T04:29:13.627842+0000","last_fullsized":"2023-10-09T04:29:13.627842+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:19.948362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.f","version":"123'9422","reported_seq":9450,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:35.284185+0000","last_change":"2023-10-09T04:28:27.611157+0000","last_active":"2023-10-09T04:29:35.284185+0000","last_peered":"2023-10-09T04:29:35.284185+0000","last_clean":"2023-10-09T04:28:26.566546+0000","last_became_active":"2023-10-09T04:28:27.611157+0000","last_became_peered":"2023-10-09T04:28:27.611157+0000","last_unstale":"2023-10-09T04:29:35.284185+0000","last_undegraded":"2023-10-09T04:28:27.598747+0000","last_fullsized":"2023-10-09T04:28:27.598504+0000","mapping_epoch":114,"log_start":"110'1100","ondisk_log_start":"110'1100","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8322,"log_dups_size":0,"ondisk_log_size":8322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307494912,"num_objects":4692,"num_object_clones":0,"num_object_copies":14076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4692,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9422,"num_write_kb":451648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4692}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.655007+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:29:13.655007+0000","last_peered":"2023-10-09T04:29:13.655007+0000","last_clean":"2023-10-09T04:29:13.655007+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:29:13.655007+0000","last_undegraded":"2023-10-09T04:29:13.655007+0000","last_fullsized":"2023-10-09T04:29:13.655007+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.655858+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:29:13.655858+0000","last_peered":"2023-10-09T04:29:13.655858+0000","last_clean":"2023-10-09T04:29:13.655858+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:29:13.655858+0000","last_undegraded":"2023-10-09T04:29:13.655858+0000","last_fullsized":"2023-10-09T04:29:13.655858+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"123'9718","reported_seq":9877,"reported_epoch":123,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T04:29:37.026920+0000","last_change":"2023-10-09T04:29:14.007034+0000","last_active":"2023-10-09T04:29:37.026920+0000","last_peered":"2023-10-09T04:29:37.026920+0000","last_clean":"2023-10-09T04:28:26.567473+0000","last_became_active":"2023-10-09T04:29:13.728615+0000","last_became_peered":"2023-10-09T04:29:13.728615+0000","last_unstale":"2023-10-09T04:29:37.026920+0000","last_undegraded":"2023-10-09T04:29:13.649108+0000","last_fullsized":"2023-10-09T04:29:13.627834+0000","mapping_epoch":121,"log_start":"110'1200","ondisk_log_start":"110'1200","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8518,"log_dups_size":0,"ondisk_log_size":8518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:46.887557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4625,"num_objects_misplaced":4672,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9718,"num_write_kb":460480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":3932160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":4672}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":82,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677269+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:29:13.677269+0000","last_peered":"2023-10-09T04:29:13.677269+0000","last_clean":"2023-10-09T04:29:13.677269+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:29:13.677269+0000","last_undegraded":"2023-10-09T04:29:13.677269+0000","last_fullsized":"2023-10-09T04:29:13.677269+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:17.679999+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:29:17.679999+0000","last_peered":"2023-10-09T04:29:17.679999+0000","last_clean":"2023-10-09T04:29:17.679999+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:29:17.679999+0000","last_undegraded":"2023-10-09T04:29:17.679999+0000","last_fullsized":"2023-10-09T04:29:17.679999+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":494,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677945+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:29:13.677945+0000","last_peered":"2023-10-09T04:29:13.677945+0000","last_clean":"2023-10-09T04:29:13.677945+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:29:13.677945+0000","last_undegraded":"2023-10-09T04:29:13.677945+0000","last_fullsized":"2023-10-09T04:29:13.677945+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":93,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.628042+0000","last_change":"2023-10-09T04:28:51.527252+0000","last_active":"2023-10-09T04:29:13.628042+0000","last_peered":"2023-10-09T04:29:13.628042+0000","last_clean":"2023-10-09T04:23:18.768463+0000","last_became_active":"2023-10-09T04:28:51.527252+0000","last_became_peered":"2023-10-09T04:28:51.527252+0000","last_unstale":"2023-10-09T04:29:13.628042+0000","last_undegraded":"2023-10-09T04:29:13.628042+0000","last_fullsized":"2023-10-09T04:28:51.524832+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:29.101177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":60,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.677608+0000","last_change":"2023-10-09T04:28:27.600497+0000","last_active":"2023-10-09T04:29:13.677608+0000","last_peered":"2023-10-09T04:29:13.677608+0000","last_clean":"2023-10-09T04:28:19.561863+0000","last_became_active":"2023-10-09T04:28:27.600497+0000","last_became_peered":"2023-10-09T04:28:27.600497+0000","last_unstale":"2023-10-09T04:29:13.677608+0000","last_undegraded":"2023-10-09T04:29:13.677608+0000","last_fullsized":"2023-10-09T04:28:27.591503+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:26.054422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"123'9461","reported_seq":9498,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:37.017095+0000","last_change":"2023-10-09T04:28:27.644798+0000","last_active":"2023-10-09T04:29:37.017095+0000","last_peered":"2023-10-09T04:29:37.017095+0000","last_clean":"2023-10-09T04:28:26.605951+0000","last_became_active":"2023-10-09T04:28:27.624767+0000","last_became_peered":"2023-10-09T04:28:27.624767+0000","last_unstale":"2023-10-09T04:29:37.017095+0000","last_undegraded":"2023-10-09T04:28:27.603721+0000","last_fullsized":"2023-10-09T04:28:27.597855+0000","mapping_epoch":114,"log_start":"110'800","ondisk_log_start":"110'800","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:11.563549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307036160,"num_objects":4685,"num_object_clones":0,"num_object_copies":14055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4685,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9461,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":4685}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"123'9505","reported_seq":10210,"reported_epoch":123,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:29:37.045327+0000","last_change":"2023-10-09T04:29:14.031224+0000","last_active":"2023-10-09T04:29:37.045327+0000","last_peered":"2023-10-09T04:29:37.045327+0000","last_clean":"2023-10-09T04:29:11.554878+0000","last_became_active":"2023-10-09T04:29:13.976505+0000","last_became_peered":"2023-10-09T04:29:13.976505+0000","last_unstale":"2023-10-09T04:29:37.045327+0000","last_undegraded":"2023-10-09T04:29:13.673293+0000","last_fullsized":"2023-10-09T04:29:13.625951+0000","mapping_epoch":121,"log_start":"110'900","ondisk_log_start":"110'900","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:39.303553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298647552,"num_objects":4557,"num_object_clones":0,"num_object_copies":13671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":755,"num_objects_misplaced":4556,"num_objects_unfound":0,"num_objects_dirty":4557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9505,"num_write_kb":449984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21954560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1","objects":-754},{"shards":"0,1,4","objects":4556}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":582,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.677523+0000","last_change":"2023-10-09T04:28:27.605360+0000","last_active":"2023-10-09T04:29:13.677523+0000","last_peered":"2023-10-09T04:29:13.677523+0000","last_clean":"2023-10-09T04:13:52.717465+0000","last_became_active":"2023-10-09T04:28:27.605360+0000","last_became_peered":"2023-10-09T04:28:27.605360+0000","last_unstale":"2023-10-09T04:29:13.677523+0000","last_undegraded":"2023-10-09T04:28:27.591227+0000","last_fullsized":"2023-10-09T04:28:27.591053+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:40.508309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.654769+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:29:13.654769+0000","last_peered":"2023-10-09T04:29:13.654769+0000","last_clean":"2023-10-09T04:29:13.654769+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:29:13.654769+0000","last_undegraded":"2023-10-09T04:29:13.654769+0000","last_fullsized":"2023-10-09T04:29:13.654769+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.678031+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:29:13.678031+0000","last_peered":"2023-10-09T04:29:13.678031+0000","last_clean":"2023-10-09T04:29:13.678031+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:29:13.678031+0000","last_undegraded":"2023-10-09T04:29:13.678031+0000","last_fullsized":"2023-10-09T04:29:13.678031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"123'9663","reported_seq":9695,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:37.090665+0000","last_change":"2023-10-09T04:28:27.609226+0000","last_active":"2023-10-09T04:29:37.090665+0000","last_peered":"2023-10-09T04:29:37.090665+0000","last_clean":"2023-10-09T04:28:26.551404+0000","last_became_active":"2023-10-09T04:28:27.609226+0000","last_became_peered":"2023-10-09T04:28:27.609226+0000","last_unstale":"2023-10-09T04:29:37.090665+0000","last_undegraded":"2023-10-09T04:28:27.592244+0000","last_fullsized":"2023-10-09T04:28:27.592122+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":9663,"log_dups_size":0,"ondisk_log_size":9663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:20.367605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300089344,"num_objects":4579,"num_object_clones":0,"num_object_copies":13737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9663,"num_write_kb":455744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":4579}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1230,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:17.674400+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:29:17.674400+0000","last_peered":"2023-10-09T04:29:17.674400+0000","last_clean":"2023-10-09T04:29:17.674400+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:29:17.674400+0000","last_undegraded":"2023-10-09T04:29:17.674400+0000","last_fullsized":"2023-10-09T04:29:17.674400+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.677550+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:29:13.677550+0000","last_peered":"2023-10-09T04:29:13.677550+0000","last_clean":"2023-10-09T04:29:13.677550+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:29:13.677550+0000","last_undegraded":"2023-10-09T04:29:13.677550+0000","last_fullsized":"2023-10-09T04:29:13.677550+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":105,"reported_epoch":123,"state":"active+undersized","last_fresh":"2023-10-09T04:29:17.679810+0000","last_change":"2023-10-09T04:28:27.599081+0000","last_active":"2023-10-09T04:29:17.679810+0000","last_peered":"2023-10-09T04:29:17.679810+0000","last_clean":"2023-10-09T04:28:19.562734+0000","last_became_active":"2023-10-09T04:28:27.599081+0000","last_became_peered":"2023-10-09T04:28:27.599081+0000","last_unstale":"2023-10-09T04:29:17.679810+0000","last_undegraded":"2023-10-09T04:29:17.679810+0000","last_fullsized":"2023-10-09T04:28:27.592506+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:57.226039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":558,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.628082+0000","last_change":"2023-10-09T04:28:27.614054+0000","last_active":"2023-10-09T04:29:13.628082+0000","last_peered":"2023-10-09T04:29:13.628082+0000","last_clean":"2023-10-09T04:13:52.699710+0000","last_became_active":"2023-10-09T04:28:27.614054+0000","last_became_peered":"2023-10-09T04:28:27.614054+0000","last_unstale":"2023-10-09T04:29:13.628082+0000","last_undegraded":"2023-10-09T04:28:27.594108+0000","last_fullsized":"2023-10-09T04:28:27.593983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:21.875430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":325,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.752387+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:29:13.752387+0000","last_peered":"2023-10-09T04:29:13.752387+0000","last_clean":"2023-10-09T04:29:13.752387+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:29:13.752387+0000","last_undegraded":"2023-10-09T04:29:13.752387+0000","last_fullsized":"2023-10-09T04:29:13.752387+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.750873+0000","last_change":"2023-10-09T04:28:51.527731+0000","last_active":"2023-10-09T04:29:13.750873+0000","last_peered":"2023-10-09T04:29:13.750873+0000","last_clean":"2023-10-09T04:28:19.566551+0000","last_became_active":"2023-10-09T04:28:51.527731+0000","last_became_peered":"2023-10-09T04:28:51.527731+0000","last_unstale":"2023-10-09T04:29:13.750873+0000","last_undegraded":"2023-10-09T04:29:13.750873+0000","last_fullsized":"2023-10-09T04:28:51.525113+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.960979+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:29:13.960979+0000","last_peered":"2023-10-09T04:29:13.960979+0000","last_clean":"2023-10-09T04:29:13.960979+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:29:13.960979+0000","last_undegraded":"2023-10-09T04:29:13.960979+0000","last_fullsized":"2023-10-09T04:29:13.960979+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"123'9486","reported_seq":9516,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:35.249235+0000","last_change":"2023-10-09T04:28:27.609668+0000","last_active":"2023-10-09T04:29:35.249235+0000","last_peered":"2023-10-09T04:29:35.249235+0000","last_clean":"2023-10-09T04:28:26.580793+0000","last_became_active":"2023-10-09T04:28:27.609668+0000","last_became_peered":"2023-10-09T04:28:27.609668+0000","last_unstale":"2023-10-09T04:29:35.249235+0000","last_undegraded":"2023-10-09T04:28:27.605975+0000","last_fullsized":"2023-10-09T04:28:27.605821+0000","mapping_epoch":114,"log_start":"110'900","ondisk_log_start":"110'900","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8586,"log_dups_size":0,"ondisk_log_size":8586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309854208,"num_objects":4728,"num_object_clones":0,"num_object_copies":14184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9486,"num_write_kb":454848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":4728}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"123'9458","reported_seq":9488,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:34.045045+0000","last_change":"2023-10-09T04:28:27.600510+0000","last_active":"2023-10-09T04:29:34.045045+0000","last_peered":"2023-10-09T04:29:34.045045+0000","last_clean":"2023-10-09T04:28:26.567988+0000","last_became_active":"2023-10-09T04:28:27.600510+0000","last_became_peered":"2023-10-09T04:28:27.600510+0000","last_unstale":"2023-10-09T04:29:34.045045+0000","last_undegraded":"2023-10-09T04:28:27.591765+0000","last_fullsized":"2023-10-09T04:28:27.591626+0000","mapping_epoch":114,"log_start":"110'1300","ondisk_log_start":"110'1300","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:02.703552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317194240,"num_objects":4840,"num_object_clones":0,"num_object_copies":14520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9458,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":4840}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":667,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.752414+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:29:13.752414+0000","last_peered":"2023-10-09T04:29:13.752414+0000","last_clean":"2023-10-09T04:29:13.752414+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:29:13.752414+0000","last_undegraded":"2023-10-09T04:29:13.752414+0000","last_fullsized":"2023-10-09T04:29:13.752414+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":84,"reported_epoch":122,"state":"active+clean+remapped","last_fresh":"2023-10-09T04:29:13.768123+0000","last_change":"2023-10-09T04:29:13.767749+0000","last_active":"2023-10-09T04:29:13.768123+0000","last_peered":"2023-10-09T04:29:13.768123+0000","last_clean":"2023-10-09T04:29:13.768123+0000","last_became_active":"2023-10-09T04:29:13.767307+0000","last_became_peered":"2023-10-09T04:29:13.767307+0000","last_unstale":"2023-10-09T04:29:13.768123+0000","last_undegraded":"2023-10-09T04:29:13.768123+0000","last_fullsized":"2023-10-09T04:29:13.768123+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:30.355543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.627914+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:29:13.627914+0000","last_peered":"2023-10-09T04:29:13.627914+0000","last_clean":"2023-10-09T04:29:13.627914+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:29:13.627914+0000","last_undegraded":"2023-10-09T04:29:13.627914+0000","last_fullsized":"2023-10-09T04:29:13.627914+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":336,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753297+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:29:13.753297+0000","last_peered":"2023-10-09T04:29:13.753297+0000","last_clean":"2023-10-09T04:29:13.753297+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:29:13.753297+0000","last_undegraded":"2023-10-09T04:29:13.753297+0000","last_fullsized":"2023-10-09T04:29:13.753297+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.678300+0000","last_change":"2023-10-09T04:28:27.599163+0000","last_active":"2023-10-09T04:29:13.678300+0000","last_peered":"2023-10-09T04:29:13.678300+0000","last_clean":"2023-10-09T04:28:19.563114+0000","last_became_active":"2023-10-09T04:28:27.599163+0000","last_became_peered":"2023-10-09T04:28:27.599163+0000","last_unstale":"2023-10-09T04:29:13.678300+0000","last_undegraded":"2023-10-09T04:29:13.678300+0000","last_fullsized":"2023-10-09T04:28:27.591374+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:37.443801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.627050+0000","last_change":"2023-10-09T04:28:27.610457+0000","last_active":"2023-10-09T04:29:13.627050+0000","last_peered":"2023-10-09T04:29:13.627050+0000","last_clean":"2023-10-09T04:28:19.571509+0000","last_became_active":"2023-10-09T04:28:27.610457+0000","last_became_peered":"2023-10-09T04:28:27.610457+0000","last_unstale":"2023-10-09T04:29:13.627050+0000","last_undegraded":"2023-10-09T04:29:13.627050+0000","last_fullsized":"2023-10-09T04:28:27.598190+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:23.684166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"123'9651","reported_seq":9680,"reported_epoch":123,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:35.258633+0000","last_change":"2023-10-09T04:28:27.608939+0000","last_active":"2023-10-09T04:29:35.258633+0000","last_peered":"2023-10-09T04:29:35.258633+0000","last_clean":"2023-10-09T04:28:26.550627+0000","last_became_active":"2023-10-09T04:28:27.608939+0000","last_became_peered":"2023-10-09T04:28:27.608939+0000","last_unstale":"2023-10-09T04:29:35.258633+0000","last_undegraded":"2023-10-09T04:28:27.603725+0000","last_fullsized":"2023-10-09T04:28:27.603461+0000","mapping_epoch":114,"log_start":"110'800","ondisk_log_start":"110'800","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8851,"log_dups_size":0,"ondisk_log_size":8851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314114048,"num_objects":4793,"num_object_clones":0,"num_object_copies":14379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4793,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9651,"num_write_kb":462208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4793}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":37,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.753154+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:29:13.753154+0000","last_peered":"2023-10-09T04:29:13.753154+0000","last_clean":"2023-10-09T04:29:13.753154+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:29:13.753154+0000","last_undegraded":"2023-10-09T04:29:13.753154+0000","last_fullsized":"2023-10-09T04:29:13.753154+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":345,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:17.673884+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:29:17.673884+0000","last_peered":"2023-10-09T04:29:17.673884+0000","last_clean":"2023-10-09T04:29:17.673884+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:29:17.673884+0000","last_undegraded":"2023-10-09T04:29:17.673884+0000","last_fullsized":"2023-10-09T04:29:17.673884+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.752186+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:29:13.752186+0000","last_peered":"2023-10-09T04:29:13.752186+0000","last_clean":"2023-10-09T04:29:13.752186+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:29:13.752186+0000","last_undegraded":"2023-10-09T04:29:13.752186+0000","last_fullsized":"2023-10-09T04:29:13.752186+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"123'9587","reported_seq":9612,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:35.292291+0000","last_change":"2023-10-09T04:26:55.436958+0000","last_active":"2023-10-09T04:29:35.292291+0000","last_peered":"2023-10-09T04:29:35.292291+0000","last_clean":"2023-10-09T04:29:35.292291+0000","last_became_active":"2023-10-09T04:26:55.436284+0000","last_became_peered":"2023-10-09T04:26:55.436284+0000","last_unstale":"2023-10-09T04:29:35.292291+0000","last_undegraded":"2023-10-09T04:29:35.292291+0000","last_fullsized":"2023-10-09T04:29:35.292291+0000","mapping_epoch":106,"log_start":"110'1200","ondisk_log_start":"110'1200","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9587,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":170,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.751509+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:29:13.751509+0000","last_peered":"2023-10-09T04:29:13.751509+0000","last_clean":"2023-10-09T04:29:13.751509+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:29:13.751509+0000","last_undegraded":"2023-10-09T04:29:13.751509+0000","last_fullsized":"2023-10-09T04:29:13.751509+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":78,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.627783+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:29:13.627783+0000","last_peered":"2023-10-09T04:29:13.627783+0000","last_clean":"2023-10-09T04:29:13.627783+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:29:13.627783+0000","last_undegraded":"2023-10-09T04:29:13.627783+0000","last_fullsized":"2023-10-09T04:29:13.627783+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.655801+0000","last_change":"2023-10-09T04:28:27.605015+0000","last_active":"2023-10-09T04:29:13.655801+0000","last_peered":"2023-10-09T04:29:13.655801+0000","last_clean":"2023-10-09T04:28:19.571550+0000","last_became_active":"2023-10-09T04:28:27.605015+0000","last_became_peered":"2023-10-09T04:28:27.605015+0000","last_unstale":"2023-10-09T04:29:13.655801+0000","last_undegraded":"2023-10-09T04:29:13.655801+0000","last_fullsized":"2023-10-09T04:28:27.598518+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"123'9387","reported_seq":9420,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T04:29:34.020092+0000","last_change":"2023-10-09T04:28:51.540229+0000","last_active":"2023-10-09T04:29:34.020092+0000","last_peered":"2023-10-09T04:29:34.020092+0000","last_clean":"2023-10-09T04:29:34.020092+0000","last_became_active":"2023-10-09T04:28:51.530787+0000","last_became_peered":"2023-10-09T04:28:51.530787+0000","last_unstale":"2023-10-09T04:29:34.020092+0000","last_undegraded":"2023-10-09T04:29:34.020092+0000","last_fullsized":"2023-10-09T04:29:34.020092+0000","mapping_epoch":117,"log_start":"110'1200","ondisk_log_start":"110'1200","created":106,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:20.235639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9387,"num_write_kb":454848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":62,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.626936+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:29:13.626936+0000","last_peered":"2023-10-09T04:29:13.626936+0000","last_clean":"2023-10-09T04:29:13.626936+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:29:13.626936+0000","last_undegraded":"2023-10-09T04:29:13.626936+0000","last_fullsized":"2023-10-09T04:29:13.626936+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":122,"state":"active+undersized","last_fresh":"2023-10-09T04:29:13.627423+0000","last_change":"2023-10-09T04:28:27.600435+0000","last_active":"2023-10-09T04:29:13.627423+0000","last_peered":"2023-10-09T04:29:13.627423+0000","last_clean":"2023-10-09T04:28:19.572818+0000","last_became_active":"2023-10-09T04:28:27.600435+0000","last_became_peered":"2023-10-09T04:28:27.600435+0000","last_unstale":"2023-10-09T04:29:13.627423+0000","last_undegraded":"2023-10-09T04:29:13.627423+0000","last_fullsized":"2023-10-09T04:28:27.592390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:45.015770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":28,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.655446+0000","last_change":"2023-10-09T04:28:27.600816+0000","last_active":"2023-10-09T04:29:13.655446+0000","last_peered":"2023-10-09T04:29:13.655446+0000","last_clean":"2023-10-09T04:11:28.922252+0000","last_became_active":"2023-10-09T04:28:27.600816+0000","last_became_peered":"2023-10-09T04:28:27.600816+0000","last_unstale":"2023-10-09T04:29:13.655446+0000","last_undegraded":"2023-10-09T04:28:27.597550+0000","last_fullsized":"2023-10-09T04:28:27.597399+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.862541+0000","last_change":"2023-10-09T04:29:12.632856+0000","last_active":"2023-10-09T04:29:13.862541+0000","last_peered":"2023-10-09T04:29:13.862541+0000","last_clean":"2023-10-09T04:29:13.862541+0000","last_became_active":"2023-10-09T04:29:12.632465+0000","last_became_peered":"2023-10-09T04:29:12.632465+0000","last_unstale":"2023-10-09T04:29:13.862541+0000","last_undegraded":"2023-10-09T04:29:13.862541+0000","last_fullsized":"2023-10-09T04:29:13.862541+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:26.854929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":28,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.655616+0000","last_change":"2023-10-09T04:28:27.606662+0000","last_active":"2023-10-09T04:29:13.655616+0000","last_peered":"2023-10-09T04:29:13.655616+0000","last_clean":"2023-10-09T04:11:28.923128+0000","last_became_active":"2023-10-09T04:28:27.606662+0000","last_became_peered":"2023-10-09T04:28:27.606662+0000","last_unstale":"2023-10-09T04:29:13.655616+0000","last_undegraded":"2023-10-09T04:28:27.599069+0000","last_fullsized":"2023-10-09T04:28:27.598215+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":89,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.625784+0000","last_change":"2023-10-09T04:29:12.632656+0000","last_active":"2023-10-09T04:29:13.625784+0000","last_peered":"2023-10-09T04:29:13.625784+0000","last_clean":"2023-10-09T04:29:13.625784+0000","last_became_active":"2023-10-09T04:29:12.632273+0000","last_became_peered":"2023-10-09T04:29:12.632273+0000","last_unstale":"2023-10-09T04:29:13.625784+0000","last_undegraded":"2023-10-09T04:29:13.625784+0000","last_fullsized":"2023-10-09T04:29:13.625784+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:27.569523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":26,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.751511+0000","last_change":"2023-10-09T04:28:51.529660+0000","last_active":"2023-10-09T04:29:13.751511+0000","last_peered":"2023-10-09T04:29:13.751511+0000","last_clean":"2023-10-09T04:29:13.751511+0000","last_became_active":"2023-10-09T04:28:51.529256+0000","last_became_peered":"2023-10-09T04:28:51.529256+0000","last_unstale":"2023-10-09T04:29:13.751511+0000","last_undegraded":"2023-10-09T04:29:13.751511+0000","last_fullsized":"2023-10-09T04:29:13.751511+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.626059+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:29:13.626059+0000","last_peered":"2023-10-09T04:29:13.626059+0000","last_clean":"2023-10-09T04:29:13.626059+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:29:13.626059+0000","last_undegraded":"2023-10-09T04:29:13.626059+0000","last_fullsized":"2023-10-09T04:29:13.626059+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":84,"reported_epoch":122,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:29:13.628310+0000","last_change":"2023-10-09T04:28:27.606014+0000","last_active":"2023-10-09T04:29:13.628310+0000","last_peered":"2023-10-09T04:29:13.628310+0000","last_clean":"2023-10-09T04:28:19.571450+0000","last_became_active":"2023-10-09T04:28:27.606014+0000","last_became_peered":"2023-10-09T04:28:27.606014+0000","last_unstale":"2023-10-09T04:29:13.628310+0000","last_undegraded":"2023-10-09T04:28:27.597046+0000","last_fullsized":"2023-10-09T04:28:27.594857+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:07.466276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T04:29:13.654318+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:29:13.654318+0000","last_peered":"2023-10-09T04:29:13.654318+0000","last_clean":"2023-10-09T04:29:13.654318+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:29:13.654318+0000","last_undegraded":"2023-10-09T04:29:13.654318+0000","last_fullsized":"2023-10-09T04:29:13.654318+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4931125272,"num_objects":75244,"num_object_clones":0,"num_object_copies":225732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":42963,"num_objects_misplaced":9228,"num_objects_unfound":0,"num_objects_dirty":75244,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153214,"num_write_kb":7310593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15645876224,"data_stored":15645868080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141314,"ondisk_log_size":141314,"up":39,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13247,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430462,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1832972,"kb_used_data":1535748,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91912180,"statfs":{"total":95995035648,"available":94118072320,"internally_reserved":0,"allocated":1572605952,"data_stored":1572085903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.96099999999999997,"15min":0.91600000000000004},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.3230000000000004,"5min":5.3230000000000004,"15min":5.3230000000000004},"last":0.371},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.218,"15min":1.014},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.6310000000000002,"5min":7.6310000000000002,"15min":7.6310000000000002},"last":0.41899999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.0369999999999999,"15min":0.96599999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.282,"5min":7.282,"15min":7.282},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.494,"5min":1.1259999999999999,"15min":0.999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.6580000000000004,"5min":5.6580000000000004,"15min":5.6580000000000004},"last":0.54800000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":1.0269999999999999,"15min":0.95899999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.0379999999999998,"5min":4.8310000000000004,"15min":4.8310000000000004},"last":0.442},{"interface":"front","average":{"1min":1.415,"5min":1.0960000000000001,"15min":1.026},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.3940000000000001,"5min":5.3940000000000001,"15min":5.3940000000000001},"last":0.69699999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:29:08 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.1220000000000001,"15min":1.0580000000000001},"min":{"1min":0.42599999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.1820000000000004,"5min":5.1820000000000004,"15min":5.1820000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0569999999999999,"5min":1.2749999999999999,"15min":1.117},"min":{"1min":0.40600000000000003,"5min":0.37,"15min":0.37},"max":{"1min":7.3399999999999999,"5min":7.3399999999999999,"15min":7.3399999999999999},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:29:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.262,"15min":1.0289999999999999},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.7969999999999997,"5min":5.7969999999999997,"15min":5.7969999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.3,"5min":1.131,"15min":1.02},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1529999999999996,"5min":5.2430000000000003,"15min":5.2430000000000003},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.0600000000000001,"15min":1.0089999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.0209999999999999,"5min":4.0209999999999999,"15min":4.0209999999999999},"last":5.4470000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.2,"15min":1.0800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":1.9219999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:29:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.2150000000000001,"15min":1.105},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.6870000000000003,"5min":5.9820000000000002,"15min":5.9820000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.109,"15min":1.0669999999999999},"min":{"1min":0.373,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.1219999999999999,"5min":3.1219999999999999,"15min":3.1219999999999999},"last":0.50600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626695,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527872,"kb_used_data":1226552,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92217280,"statfs":{"total":95995035648,"available":94430494720,"internally_reserved":0,"allocated":1255989248,"data_stored":1255481336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1349999999999998,"5min":1.3280000000000001,"15min":1.038},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":17.591000000000001,"5min":17.591000000000001,"15min":17.591000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.2030000000000001,"15min":0.97199999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":17.506,"5min":17.506,"15min":17.506},"last":0.441}]},{"osd":1,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.073,"15min":0.95399999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.2690000000000001,"5min":8.2690000000000001,"15min":8.2690000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.427,"5min":1.016,"15min":0.93700000000000006},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.099,"5min":10.099,"15min":10.099},"last":2.6669999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:29:23 2023","interfaces":[{"interface":"back","average":{"1min":11.032,"5min":3.4740000000000002,"15min":1.8049999999999999},"min":{"1min":0.32900000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":141.93899999999999,"5min":141.93899999999999,"15min":141.93899999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":9.8740000000000006,"5min":3.137,"15min":1.661},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":141.898,"5min":141.898,"15min":141.898},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.0960000000000001,"15min":0.97599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.107,"15min":1.0620000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":6.0519999999999996,"5min":6.0519999999999996,"15min":6.0519999999999996},"last":0.48999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:29:23 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1479999999999999,"15min":0.98299999999999998},"min":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.0379999999999998,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.097,"15min":0.999},"min":{"1min":0.432,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":3.3340000000000001,"5min":11.018000000000001,"15min":11.018000000000001},"last":0.55800000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.0509999999999999,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.7120000000000002,"5min":3.7120000000000002,"15min":3.9180000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.0670000000000002,"5min":1.242,"15min":1.0780000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.05,"15min":1.0620000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.7320000000000002,"5min":5.0339999999999998,"15min":5.0339999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.788,"5min":1.2090000000000001,"15min":1.1779999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":8.9139999999999997,"5min":8.9139999999999997,"15min":8.9139999999999997},"last":0.74399999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958611,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1620508,"kb_used_data":1544468,"kb_used_omap":6,"kb_used_meta":75961,"kb_avail":92124644,"statfs":{"total":95995035648,"available":94335635456,"internally_reserved":0,"allocated":1581535232,"data_stored":1580944121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":77784076},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.8250000000000002,"15min":1.391},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.9320000000000004,"5min":158.435,"15min":158.435},"last":6.1820000000000004},{"interface":"front","average":{"1min":2.7879999999999998,"5min":2.9129999999999998,"15min":1.4690000000000001},"min":{"1min":0.54300000000000004,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.117,"5min":159.38999999999999,"15min":159.38999999999999},"last":6.093}]},{"osd":2,"last update":"Mon Oct 9 04:28:36 2023","interfaces":[{"interface":"back","average":{"1min":11.096,"5min":2.8730000000000002,"15min":1.409},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":157.70400000000001,"5min":157.70400000000001,"15min":157.70400000000001},"last":1.0089999999999999},{"interface":"front","average":{"1min":11.286,"5min":2.948,"15min":1.506},"min":{"1min":0.35499999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":157.64099999999999,"5min":157.64099999999999,"15min":157.64099999999999},"last":1.528}]},{"osd":3,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.0409999999999999,"15min":0.89700000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.3079999999999998,"5min":5.3079999999999998,"15min":5.3079999999999998},"last":0.371},{"interface":"front","average":{"1min":1.379,"5min":0.876,"15min":0.748},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":6.8339999999999996,"5min":6.8339999999999996,"15min":6.8339999999999996},"last":0.31900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.073,"15min":0.97099999999999997},"min":{"1min":0.35899999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.29,"5min":4.29,"15min":4.29},"last":2.6150000000000002},{"interface":"front","average":{"1min":1.7,"5min":1.1839999999999999,"15min":1.0920000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.3109999999999999,"5min":4.3109999999999999,"15min":4.3109999999999999},"last":0.40999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.9449999999999998,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.99099999999999999,"15min":0.98899999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7069999999999999,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.0489999999999999,"15min":1.0369999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.258,"5min":5.258,"15min":5.258},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.0860000000000001,"15min":1.046},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":4.3230000000000004,"5min":4.3230000000000004,"15min":4.3230000000000004},"last":2.5449999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.155,"15min":1.0549999999999999},"min":{"1min":0.42199999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.3419999999999996,"5min":4.3419999999999996,"15min":4.3419999999999996},"last":2.649},{"interface":"front","average":{"1min":1.7969999999999999,"5min":1.266,"15min":1.1699999999999999},"min":{"1min":0.45600000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":6.3310000000000004,"5min":7.0999999999999996,"15min":7.0999999999999996},"last":0.48899999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122137,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2513960,"kb_used_data":2396000,"kb_used_omap":9,"kb_used_meta":117878,"kb_avail":91231192,"statfs":{"total":95995035648,"available":93420740608,"internally_reserved":0,"allocated":2453504000,"data_stored":2452903158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":120707569},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.628,"15min":0.63900000000000001},"min":{"1min":0.20799999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.452,"5min":3.452,"15min":3.452},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.222,"5min":0.79900000000000004,"15min":0.71499999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":3.7719999999999998,"5min":3.7719999999999998,"15min":3.7719999999999998},"last":5.1390000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:29:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.4930000000000001,"15min":0.93899999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.625,"5min":37.453000000000003,"15min":37.453000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":2.077,"5min":1.2549999999999999,"15min":0.94099999999999995},"min":{"1min":0.29599999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.7990000000000004,"5min":25.178999999999998,"15min":25.178999999999998},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:29:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.83599999999999997,"15min":0.82899999999999996},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.6689999999999996,"5min":6.6689999999999996,"15min":6.6689999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.4670000000000001,"5min":0.94499999999999995,"15min":0.78700000000000003},"min":{"1min":0.34799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.0899999999999999,"5min":7.5250000000000004,"15min":7.5250000000000004},"last":0.56899999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:29:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.417,"15min":1.1919999999999999},"min":{"1min":0.33800000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.1080000000000005,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":1.506},{"interface":"front","average":{"1min":1.5860000000000001,"5min":1.0980000000000001,"15min":1.0609999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":7.5880000000000001,"5min":7.5880000000000001,"15min":7.5880000000000001},"last":0.621}]},{"osd":5,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.034,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":3.6339999999999999,"5min":3.6339999999999999,"15min":3.6339999999999999},"last":0.441},{"interface":"front","average":{"1min":1.238,"5min":1.0369999999999999,"15min":1.038},"min":{"1min":0.65500000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":3.343,"5min":3.343,"15min":3.343},"last":2.0150000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:29:31 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.1220000000000001,"15min":1.0649999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.4260000000000002,"5min":7.508,"15min":7.508},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.196,"5min":1.304,"15min":1.1140000000000001},"min":{"1min":0.42999999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.6260000000000003,"5min":23.994,"15min":23.994},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:29:31 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":1.454,"15min":1.2390000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.5850000000000009,"5min":10.528,"15min":10.528},"last":0.79400000000000004},{"interface":"front","average":{"1min":3.0670000000000002,"5min":1.6519999999999999,"15min":1.3140000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":9.1449999999999996,"5min":9.7029999999999994,"15min":9.7029999999999994},"last":1.536}]}]},{"osd":2,"up_from":98,"seq":420906795086,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1909520,"kb_used_data":1797448,"kb_used_omap":11,"kb_used_meta":112052,"kb_avail":91835632,"statfs":{"total":95995035648,"available":94039687168,"internally_reserved":0,"allocated":1840586752,"data_stored":1840004975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":114741290},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:27 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":0.997,"15min":0.80100000000000005},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":7.577,"5min":7.577,"15min":7.577},"last":0.245},{"interface":"front","average":{"1min":2.3719999999999999,"5min":1.1319999999999999,"15min":0.81200000000000006},"min":{"1min":0.313,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":0.58699999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:29:27 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.032,"15min":0.83899999999999997},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.6929999999999996,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.4929999999999999,"5min":1.2569999999999999,"15min":0.89400000000000002},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.356999999999999,"5min":11.356999999999999,"15min":11.356999999999999},"last":0.69199999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:29:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.0900000000000001,"15min":0.91400000000000003},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.044,"5min":10.044,"15min":10.044},"last":0.313},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.2250000000000001,"15min":0.97199999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":7.7130000000000001,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":0.54300000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:29:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.4550000000000001,"15min":1.117},"min":{"1min":0.35799999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.6340000000000003,"5min":7.6340000000000003,"15min":7.6340000000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.802,"5min":1.6499999999999999,"15min":1.1930000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":7.6130000000000004,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0409999999999999,"15min":1.0309999999999999},"min":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.113,"5min":0.98499999999999999,"15min":0.96799999999999997},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":4.649}]},{"osd":6,"last update":"Mon Oct 9 04:29:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.6899999999999999,"15min":1.248},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":7.7729999999999997,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.431,"5min":1.1419999999999999,"15min":1.0960000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:29:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.2929999999999999,"15min":1.198},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.2430000000000003,"5min":8.2430000000000003,"15min":8.2430000000000003},"last":0.499},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.512,"15min":1.236},"min":{"1min":0.50700000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":7.9530000000000003,"5min":7.9530000000000003,"15min":7.9530000000000003},"last":0.61899999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631557,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993476,"kb_used_data":935612,"kb_used_omap":5,"kb_used_meta":57850,"kb_avail":92751676,"statfs":{"total":95995035648,"available":94977716224,"internally_reserved":0,"allocated":958066688,"data_stored":957496119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":59238574},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":5.0460000000000003,"5min":1.7290000000000001,"15min":1.0980000000000001},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":61.966999999999999,"5min":61.966999999999999,"15min":61.966999999999999},"last":0.91000000000000003},{"interface":"front","average":{"1min":5.0499999999999998,"5min":1.7310000000000001,"15min":1.0389999999999999},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":62.052999999999997,"5min":62.052999999999997,"15min":62.052999999999997},"last":0.88100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0940000000000003,"5min":1.4139999999999999,"15min":0.95399999999999996},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":61.395000000000003,"5min":61.395000000000003,"15min":61.395000000000003},"last":1.732},{"interface":"front","average":{"1min":4.1070000000000002,"5min":1.4610000000000001,"15min":1.0489999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":61.179000000000002,"5min":61.179000000000002,"15min":61.179000000000002},"last":2.1360000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":5.0270000000000001,"5min":1.7470000000000001,"15min":1.129},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":63.274999999999999,"5min":63.274999999999999,"15min":63.274999999999999},"last":0.25},{"interface":"front","average":{"1min":4.8959999999999999,"5min":1.762,"15min":1.24},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":63.107999999999997,"5min":63.107999999999997,"15min":63.107999999999997},"last":0.42899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.0229999999999999,"15min":1.0740000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.0789999999999997,"5min":4.0789999999999997,"15min":4.0789999999999997},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.4910000000000001,"5min":1.101,"15min":1.0349999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.1820000000000004,"5min":4.1820000000000004,"15min":4.1820000000000004},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.96499999999999997,"15min":1.0229999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":1.917},{"interface":"front","average":{"1min":0.872,"5min":0.997,"15min":1.056},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.996,"5min":1.996,"15min":1.996},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":0.995,"15min":1.0369999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.528,"5min":3.528,"15min":3.528},"last":1.018},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.0289999999999999,"15min":1.083},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.6930000000000001,"5min":3.6930000000000001,"15min":3.6930000000000001},"last":0.45100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.0820000000000001,"15min":1.1240000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.673,"5min":3.673,"15min":3.673},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.1519999999999999,"15min":1.214},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.948,"5min":5.3339999999999996,"15min":5.3339999999999996},"last":0.59899999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019159,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2815808,"kb_used_data":2506296,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90929344,"statfs":{"total":95995035648,"available":93111648256,"internally_reserved":0,"allocated":2566447104,"data_stored":2565938705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6950000000000001,"15min":1.1399999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":20.721,"5min":48.450000000000003,"15min":48.450000000000003},"last":0.28699999999999998},{"interface":"front","average":{"1min":2.7360000000000002,"5min":1.8169999999999999,"15min":1.2110000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":21.061,"5min":49.283000000000001,"15min":49.283000000000001},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.103,"15min":0.97599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.6189999999999998,"5min":6.1299999999999999,"15min":6.1299999999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.665,"5min":1.214,"15min":0.998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.9399999999999995,"5min":9.9399999999999995,"15min":9.9399999999999995},"last":0.69899999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:29:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":1.8360000000000001,"15min":1.2270000000000001},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":20.300000000000001,"5min":48.323,"15min":48.323},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.8809999999999998,"5min":2.069,"15min":1.329},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":20.609999999999999,"5min":48.289000000000001,"15min":48.289000000000001},"last":0.88100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:29:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.996,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":5.7220000000000004,"5min":5.7220000000000004,"15min":5.7220000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.99199999999999999,"15min":1.0549999999999999},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.0199999999999996,"5min":4.0199999999999996,"15min":4.0199999999999996},"last":0.59999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98999999999999999,"15min":0.95399999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.087,"15min":0.95399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":1.4770000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.03,"15min":0.99099999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8540000000000001,"5min":4.8540000000000001,"15min":4.8540000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.2070000000000001,"15min":1.0920000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":11.597,"5min":11.597,"15min":11.597},"last":0.373}]},{"osd":7,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":1.302,"15min":1.181},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.4279999999999999,"5min":11.725,"15min":11.725},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.22,"15min":1.143},"min":{"1min":0.40500000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.5730000000000004,"5min":6.5730000000000004,"15min":6.5730000000000004},"last":0.62}]}]},{"osd":5,"up_from":36,"seq":154618822912,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3745056,"kb_used_data":3439640,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90000096,"statfs":{"total":95995035648,"available":92160098304,"internally_reserved":0,"allocated":3522191360,"data_stored":3521698716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8919999999999999,"5min":1.474,"15min":1.073},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":53.731000000000002,"5min":53.731000000000002,"15min":53.731000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":1.5429999999999999,"15min":1.0620000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":53.222999999999999,"5min":53.222999999999999,"15min":53.222999999999999},"last":2.359}]},{"osd":1,"last update":"Mon Oct 9 04:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.98099999999999998,"15min":0.997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.252},{"interface":"front","average":{"1min":1.038,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.94699999999999995,"15min":0.94099999999999995},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":2.4430000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":2.5390000000000001,"5min":2.5390000000000001,"15min":2.5390000000000001},"last":2.4649999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.008,"15min":0.99399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":2.2789999999999999,"5min":2.2789999999999999,"15min":2.2789999999999999},"last":0.378},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.1240000000000001,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5140000000000002,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":2.4180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:43 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96299999999999997,"15min":0.94899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9100000000000001,"5min":3.9100000000000001,"15min":3.9100000000000001},"last":5.6479999999999997},{"interface":"front","average":{"1min":1.304,"5min":1.0900000000000001,"15min":1.012},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":3.9990000000000001},"last":2.395}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.998,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.9849999999999999,"5min":3.9849999999999999,"15min":3.9849999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0740000000000001,"15min":1.0660000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.847,"5min":1.847,"15min":2.5840000000000001},"last":0.32800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1379999999999999,"15min":1.113},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.431,"5min":3.992,"15min":3.992},"last":0.312},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.1120000000000001,"15min":1.095},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":1.4610000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2438397952,"data_stored":2438397952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1567031296,"data_stored":1567031296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1817448448,"data_stored":1817444376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":948895744,"data_stored":948895744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2545221632,"data_stored":2545221632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3527147520,"data_stored":3527147520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1246953472,"data_stored":1246953472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1554780160,"data_stored":1554776088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:29:41.391 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-10-09T04:29:41.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:41.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:41.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:41.780 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:41.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:41.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:41.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:41.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:42.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:42.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:42.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:42.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:42.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:42.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:42.352 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:42.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:42.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:42.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:42.616 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:42 smithi082 bash[77063]: cluster 2023-10-09T04:29:41.408061+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v409: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 41965/220050 objects degraded (19.071%); 9228/220050 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:42.616 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:42 smithi082 bash[77063]: audit 2023-10-09T04:29:42.444909+0000 mon.a (mon.0) 647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:42.616 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:42 smithi082 bash[79733]: cluster 2023-10-09T04:29:41.408061+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v409: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 41965/220050 objects degraded (19.071%); 9228/220050 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:42.616 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:42 smithi082 bash[79733]: audit 2023-10-09T04:29:42.444909+0000 mon.a (mon.0) 647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:42.642 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:42.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:42.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:42.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:42.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:42.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:42 smithi153 bash[59565]: cluster 2023-10-09T04:29:41.408061+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v409: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 41965/220050 objects degraded (19.071%); 9228/220050 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:42.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:42 smithi153 bash[59565]: audit 2023-10-09T04:29:42.444909+0000 mon.a (mon.0) 647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:42.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:42.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:43.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.039 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:43.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:43.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:43.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:43.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:43.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:43.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:29:43.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:43.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:43.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:43.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:43.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:43.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:43.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:43.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:43.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:44.093 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:44.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:44.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:44.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:44.499 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:44.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:44.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[77063]: cluster 2023-10-09T04:29:43.409103+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v410: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 40986/214656 objects degraded (19.094%); 8765/214656 objects misplaced (4.083%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[77063]: audit 2023-10-09T04:29:44.265032+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[77063]: audit 2023-10-09T04:29:44.270468+0000 mon.a (mon.0) 649 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[77063]: cluster 2023-10-09T04:29:44.441454+0000 mon.a (mon.0) 650 : cluster 3 Health check update: Degraded data redundancy: 40986/214656 objects degraded (19.094%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[79733]: cluster 2023-10-09T04:29:43.409103+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v410: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 40986/214656 objects degraded (19.094%); 8765/214656 objects misplaced (4.083%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[79733]: audit 2023-10-09T04:29:44.265032+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[79733]: audit 2023-10-09T04:29:44.270468+0000 mon.a (mon.0) 649 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:44 smithi082 bash[79733]: cluster 2023-10-09T04:29:44.441454+0000 mon.a (mon.0) 650 : cluster 3 Health check update: Degraded data redundancy: 40986/214656 objects degraded (19.094%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:44.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:44.776 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:44 smithi153 bash[59565]: cluster 2023-10-09T04:29:43.409103+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v410: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 40986/214656 objects degraded (19.094%); 8765/214656 objects misplaced (4.083%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:44.776 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:44 smithi153 bash[59565]: audit 2023-10-09T04:29:44.265032+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:44.777 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:44 smithi153 bash[59565]: audit 2023-10-09T04:29:44.270468+0000 mon.a (mon.0) 649 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.777 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:44 smithi153 bash[59565]: cluster 2023-10-09T04:29:44.441454+0000 mon.a (mon.0) 650 : cluster 3 Health check update: Degraded data redundancy: 40986/214656 objects degraded (19.094%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:44.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:44.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:44.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:44.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:44.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:45.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:45.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:45.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:45.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:45.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:45.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:45.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:45.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:45.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:45.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:45.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:45.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:45.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:45.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:45.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:45.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:45.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:46.007 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:46.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:46.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:46.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:46.213 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:46.214 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:46.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:46.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:46.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:46.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:46.509 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:46 smithi082 bash[77063]: cluster 2023-10-09T04:29:45.410093+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v411: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 40064/209175 objects degraded (19.153%); 8765/209175 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:46.509 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:46 smithi082 bash[79733]: cluster 2023-10-09T04:29:45.410093+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v411: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 40064/209175 objects degraded (19.153%); 8765/209175 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:46.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:46.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:46 smithi153 bash[59565]: cluster 2023-10-09T04:29:45.410093+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v411: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 40064/209175 objects degraded (19.153%); 8765/209175 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:29:46.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:46.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:46.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:46.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:46.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:46.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:46.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:46.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:47.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:47.109 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:47.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.237 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:47.238 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:47.246 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:47.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:47.370 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:47.476 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:47.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:47.649 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:47.798 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:47.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:47.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:47.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:48.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:48.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:48.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:48.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:48.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.492 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.524 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:48 smithi082 bash[79733]: cluster 2023-10-09T04:29:47.413695+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v412: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 39044/203730 objects degraded (19.165%); 8327/203730 objects misplaced (4.087%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:29:48.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:48.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:48.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:48.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:48.876 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:48 smithi153 bash[59565]: cluster 2023-10-09T04:29:47.413695+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v412: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 39044/203730 objects degraded (19.165%); 8327/203730 objects misplaced (4.087%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:29:48.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:48.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:48.963 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:48 smithi082 bash[77063]: cluster 2023-10-09T04:29:47.413695+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v412: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 39044/203730 objects degraded (19.165%); 8327/203730 objects misplaced (4.087%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:29:48.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:49.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:49.132 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:49.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:49.283 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:49.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:49.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:49.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:49.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.702 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:49] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T04:29:49.704 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:49.811 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:48.724294+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:48.732683+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:48.883169+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:48.892828+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:49.437514+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:49.439429+0000 mon.a (mon.0) 656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:49.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: audit 2023-10-09T04:29:49.450076+0000 mon.a (mon.0) 657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:49 smithi153 bash[59565]: cluster 2023-10-09T04:29:49.450312+0000 mon.a (mon.0) 658 : cluster 3 Health check update: Degraded data redundancy: 39044/203730 objects degraded (19.165%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:49.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:49.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:49.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:49.970 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:49.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:48.724294+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:48.732683+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:48.883169+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:48.892828+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:49.437514+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:49.439429+0000 mon.a (mon.0) 656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: audit 2023-10-09T04:29:49.450076+0000 mon.a (mon.0) 657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[77063]: cluster 2023-10-09T04:29:49.450312+0000 mon.a (mon.0) 658 : cluster 3 Health check update: Degraded data redundancy: 39044/203730 objects degraded (19.165%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:48.724294+0000 mon.a (mon.0) 651 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:48.732683+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:48.883169+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:48.892828+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:49.437514+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:49.439429+0000 mon.a (mon.0) 656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: audit 2023-10-09T04:29:49.450076+0000 mon.a (mon.0) 657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:29:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:49 smithi082 bash[79733]: cluster 2023-10-09T04:29:49.450312+0000 mon.a (mon.0) 658 : cluster 3 Health check update: Degraded data redundancy: 39044/203730 objects degraded (19.165%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:50.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:50.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:50.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:50.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:50.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:50.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:50.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:50.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:50.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:50.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:50.740 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:50 smithi153 bash[59565]: cluster 2023-10-09T04:29:49.414537+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 38815/201738 objects degraded (19.240%); 8327/201738 objects misplaced (4.128%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:29:50.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:50.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:50.868 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:50 smithi082 bash[77063]: cluster 2023-10-09T04:29:49.414537+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 38815/201738 objects degraded (19.240%); 8327/201738 objects misplaced (4.128%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:29:50.869 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:50 smithi082 bash[79733]: cluster 2023-10-09T04:29:49.414537+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 38815/201738 objects degraded (19.240%); 8327/201738 objects misplaced (4.128%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:29:50.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:50.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:50.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:51.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:51.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:51.161 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:51.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:51.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:51.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:51.421 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:51.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:51.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:51.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:51.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:51.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:51.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:51.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:51.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:51.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:52.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.110 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:52.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:52.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:52.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:52.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:52.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:52.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:52.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:52.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:52.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:52.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.885 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:52 smithi153 bash[59565]: cluster 2023-10-09T04:29:51.415198+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 38111/198267 objects degraded (19.222%); 8327/198267 objects misplaced (4.200%); 901 KiB/s, 14 objects/s recovering 2023-10-09T04:29:52.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:52.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:52.994 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:52 smithi082 bash[77063]: cluster 2023-10-09T04:29:51.415198+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 38111/198267 objects degraded (19.222%); 8327/198267 objects misplaced (4.200%); 901 KiB/s, 14 objects/s recovering 2023-10-09T04:29:52.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:52 smithi082 bash[79733]: cluster 2023-10-09T04:29:51.415198+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v414: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 38111/198267 objects degraded (19.222%); 8327/198267 objects misplaced (4.200%); 901 KiB/s, 14 objects/s recovering 2023-10-09T04:29:53.022 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:53.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:53.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:53.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:53.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:53.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:53.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:53.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:53.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:53.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:53.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:53.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:53.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:53.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:53.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:53.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:53.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:53.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:54.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:54.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:54.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:54.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:54.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:54.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:54.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:54.780 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:54.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:54.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:54 smithi153 bash[59565]: cluster 2023-10-09T04:29:53.416017+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 37063/192840 objects degraded (19.220%); 7890/192840 objects misplaced (4.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:54 smithi153 bash[59565]: cluster 2023-10-09T04:29:54.454201+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 37063/192840 objects degraded (19.220%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:54.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:54.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:55.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:55.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:55.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:55.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:55.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:54 smithi082 bash[77063]: cluster 2023-10-09T04:29:53.416017+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 37063/192840 objects degraded (19.220%); 7890/192840 objects misplaced (4.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:55.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:54 smithi082 bash[77063]: cluster 2023-10-09T04:29:54.454201+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 37063/192840 objects degraded (19.220%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:55.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:54 smithi082 bash[79733]: cluster 2023-10-09T04:29:53.416017+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v415: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 37063/192840 objects degraded (19.220%); 7890/192840 objects misplaced (4.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:55.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:54 smithi082 bash[79733]: cluster 2023-10-09T04:29:54.454201+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 37063/192840 objects degraded (19.220%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:55.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:55.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:55.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:55.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:55.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:55.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:55.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:55.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:55.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:55.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:55.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:55.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:55.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:55.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:56.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:56.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:56.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:56.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:56.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:56.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:56.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:56.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:56.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:56.591 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:56.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:56.715 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:56.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:56.851 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:56.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:56.963 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:56 smithi153 bash[59565]: cluster 2023-10-09T04:29:55.416980+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 36088/186930 objects degraded (19.306%); 7890/186930 objects misplaced (4.221%); 906 KiB/s, 14 objects/s recovering 2023-10-09T04:29:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:56.989 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:56.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:57.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.098 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:56 smithi082 bash[77063]: cluster 2023-10-09T04:29:55.416980+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 36088/186930 objects degraded (19.306%); 7890/186930 objects misplaced (4.221%); 906 KiB/s, 14 objects/s recovering 2023-10-09T04:29:57.098 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:56 smithi082 bash[79733]: cluster 2023-10-09T04:29:55.416980+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v416: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 36088/186930 objects degraded (19.306%); 7890/186930 objects misplaced (4.221%); 906 KiB/s, 14 objects/s recovering 2023-10-09T04:29:57.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:57.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:57.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:57.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:57.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:57.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:57.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:57.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:57.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:57.840 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:57.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.968 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:57.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:58.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:58.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:58.229 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:58.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:58.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:58.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:58.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:58.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:58.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:58.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:58.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:58.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:58.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:58.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:58.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:58.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:58 smithi153 bash[59565]: cluster 2023-10-09T04:29:57.417796+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v417: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 35056/181197 objects degraded (19.347%); 7402/181197 objects misplaced (4.085%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:58.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:58.994 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:58 smithi082 bash[77063]: cluster 2023-10-09T04:29:57.417796+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v417: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 35056/181197 objects degraded (19.347%); 7402/181197 objects misplaced (4.085%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:58.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:58 smithi082 bash[79733]: cluster 2023-10-09T04:29:57.417796+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v417: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 35056/181197 objects degraded (19.347%); 7402/181197 objects misplaced (4.085%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:29:59.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:59.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:59.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:59.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:59.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:59.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:59.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:59.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:59.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:59.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:59.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:59.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:59.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:59.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:59.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:59.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:59.907 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:59 smithi153 bash[59565]: audit 2023-10-09T04:29:59.259786+0000 mon.a (mon.0) 660 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:59.908 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:29:59 smithi153 bash[59565]: cluster 2023-10-09T04:29:59.456214+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 35056/181197 objects degraded (19.347%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:59.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:29:59.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:59 smithi082 bash[77063]: audit 2023-10-09T04:29:59.259786+0000 mon.a (mon.0) 660 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:29:59 smithi082 bash[77063]: cluster 2023-10-09T04:29:59.456214+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 35056/181197 objects degraded (19.347%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:59.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:59 smithi082 bash[79733]: audit 2023-10-09T04:29:59.259786+0000 mon.a (mon.0) 660 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:59.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:29:59 smithi082 bash[79733]: cluster 2023-10-09T04:29:59.456214+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 35056/181197 objects degraded (19.347%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:29:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:29:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:29:59] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:30:00.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:00.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:00.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:00.202 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:00.305 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:00.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:00.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:00.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:00.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:00.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:00.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:00.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:00.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:00.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:00.850 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:00.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:00.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:29:59.418533+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 34832/179076 objects degraded (19.451%); 7402/179076 objects misplaced (4.133%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000148+0000 mon.a (mon.0) 662 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000185+0000 mon.a (mon.0) 663 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:30:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000196+0000 mon.a (mon.0) 664 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000217+0000 mon.a (mon.0) 665 : cluster 3 osd.5 (root=default,host=smithi153) is down 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000230+0000 mon.a (mon.0) 666 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000246+0000 mon.a (mon.0) 667 : cluster 3 pg 2.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,6] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000261+0000 mon.a (mon.0) 668 : cluster 3 pg 2.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000276+0000 mon.a (mon.0) 669 : cluster 3 pg 3.2 is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000289+0000 mon.a (mon.0) 670 : cluster 3 pg 3.4 is stuck undersized for 91s, current state active+undersized, last acting [1,2] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000302+0000 mon.a (mon.0) 671 : cluster 3 pg 3.5 is stuck undersized for 67s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000313+0000 mon.a (mon.0) 672 : cluster 3 pg 3.c is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000325+0000 mon.a (mon.0) 673 : cluster 3 pg 3.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,6] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000337+0000 mon.a (mon.0) 674 : cluster 3 pg 3.10 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000363+0000 mon.a (mon.0) 675 : cluster 3 pg 3.16 is stuck undersized for 91s, current state active+undersized, last acting [7,1] 2023-10-09T04:30:00.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000377+0000 mon.a (mon.0) 676 : cluster 3 pg 3.17 is stuck undersized for 91s, current state active+undersized, last acting [0,3] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000391+0000 mon.a (mon.0) 677 : cluster 3 pg 3.1c is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,1] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000404+0000 mon.a (mon.0) 678 : cluster 3 pg 3.1d is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000417+0000 mon.a (mon.0) 679 : cluster 3 pg 3.1f is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,2] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000431+0000 mon.a (mon.0) 680 : cluster 3 pg 4.1 is stuck undersized for 91s, current state active+undersized, last acting [4,6] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000443+0000 mon.a (mon.0) 681 : cluster 3 pg 4.2 is stuck undersized for 91s, current state active+undersized, last acting [1,4] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000458+0000 mon.a (mon.0) 682 : cluster 3 pg 4.3 is stuck undersized for 91s, current state active+undersized, last acting [0,7] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000472+0000 mon.a (mon.0) 683 : cluster 3 pg 4.7 is stuck undersized for 91s, current state active+undersized, last acting [1,0] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000486+0000 mon.a (mon.0) 684 : cluster 3 pg 4.8 is stuck undersized for 91s, current state active+undersized, last acting [7,6] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000500+0000 mon.a (mon.0) 685 : cluster 3 pg 4.15 is stuck undersized for 91s, current state active+undersized, last acting [7,3] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000514+0000 mon.a (mon.0) 686 : cluster 3 pg 4.1c is stuck undersized for 91s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:00.984 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000528+0000 mon.a (mon.0) 687 : cluster 3 pg 4.1f is stuck undersized for 91s, current state active+undersized, last acting [6,1] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000541+0000 mon.a (mon.0) 688 : cluster 3 pg 5.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000563+0000 mon.a (mon.0) 689 : cluster 3 pg 5.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,6] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000604+0000 mon.a (mon.0) 690 : cluster 3 pg 5.4 is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,2] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000621+0000 mon.a (mon.0) 691 : cluster 3 pg 5.5 is active+recovering+undersized+degraded+remapped, acting [0,1] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000636+0000 mon.a (mon.0) 692 : cluster 3 pg 5.6 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000667+0000 mon.a (mon.0) 693 : cluster 3 pg 5.7 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,0] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000683+0000 mon.a (mon.0) 694 : cluster 3 pg 5.b is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,0] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000698+0000 mon.a (mon.0) 695 : cluster 3 pg 5.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000713+0000 mon.a (mon.0) 696 : cluster 3 pg 5.e is active+undersized+degraded+remapped+backfill_wait, acting [0,4] 2023-10-09T04:30:00.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:00.000727+0000 mon.a (mon.0) 697 : cluster 3 pg 5.f is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:29:59.418533+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 34832/179076 objects degraded (19.451%); 7402/179076 objects misplaced (4.133%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000148+0000 mon.a (mon.0) 662 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000185+0000 mon.a (mon.0) 663 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000196+0000 mon.a (mon.0) 664 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000217+0000 mon.a (mon.0) 665 : cluster 3 osd.5 (root=default,host=smithi153) is down 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000230+0000 mon.a (mon.0) 666 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:00.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000246+0000 mon.a (mon.0) 667 : cluster 3 pg 2.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,6] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000261+0000 mon.a (mon.0) 668 : cluster 3 pg 2.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000276+0000 mon.a (mon.0) 669 : cluster 3 pg 3.2 is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000289+0000 mon.a (mon.0) 670 : cluster 3 pg 3.4 is stuck undersized for 91s, current state active+undersized, last acting [1,2] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000302+0000 mon.a (mon.0) 671 : cluster 3 pg 3.5 is stuck undersized for 67s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000313+0000 mon.a (mon.0) 672 : cluster 3 pg 3.c is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000325+0000 mon.a (mon.0) 673 : cluster 3 pg 3.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,6] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000337+0000 mon.a (mon.0) 674 : cluster 3 pg 3.10 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000363+0000 mon.a (mon.0) 675 : cluster 3 pg 3.16 is stuck undersized for 91s, current state active+undersized, last acting [7,1] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000377+0000 mon.a (mon.0) 676 : cluster 3 pg 3.17 is stuck undersized for 91s, current state active+undersized, last acting [0,3] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000391+0000 mon.a (mon.0) 677 : cluster 3 pg 3.1c is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,1] 2023-10-09T04:30:00.987 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000404+0000 mon.a (mon.0) 678 : cluster 3 pg 3.1d is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000417+0000 mon.a (mon.0) 679 : cluster 3 pg 3.1f is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,2] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000431+0000 mon.a (mon.0) 680 : cluster 3 pg 4.1 is stuck undersized for 91s, current state active+undersized, last acting [4,6] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000443+0000 mon.a (mon.0) 681 : cluster 3 pg 4.2 is stuck undersized for 91s, current state active+undersized, last acting [1,4] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000458+0000 mon.a (mon.0) 682 : cluster 3 pg 4.3 is stuck undersized for 91s, current state active+undersized, last acting [0,7] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000472+0000 mon.a (mon.0) 683 : cluster 3 pg 4.7 is stuck undersized for 91s, current state active+undersized, last acting [1,0] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000486+0000 mon.a (mon.0) 684 : cluster 3 pg 4.8 is stuck undersized for 91s, current state active+undersized, last acting [7,6] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000500+0000 mon.a (mon.0) 685 : cluster 3 pg 4.15 is stuck undersized for 91s, current state active+undersized, last acting [7,3] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000514+0000 mon.a (mon.0) 686 : cluster 3 pg 4.1c is stuck undersized for 91s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000528+0000 mon.a (mon.0) 687 : cluster 3 pg 4.1f is stuck undersized for 91s, current state active+undersized, last acting [6,1] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000541+0000 mon.a (mon.0) 688 : cluster 3 pg 5.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:00.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000563+0000 mon.a (mon.0) 689 : cluster 3 pg 5.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,6] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000604+0000 mon.a (mon.0) 690 : cluster 3 pg 5.4 is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,2] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000621+0000 mon.a (mon.0) 691 : cluster 3 pg 5.5 is active+recovering+undersized+degraded+remapped, acting [0,1] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000636+0000 mon.a (mon.0) 692 : cluster 3 pg 5.6 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000667+0000 mon.a (mon.0) 693 : cluster 3 pg 5.7 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,0] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000683+0000 mon.a (mon.0) 694 : cluster 3 pg 5.b is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,0] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000698+0000 mon.a (mon.0) 695 : cluster 3 pg 5.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000713+0000 mon.a (mon.0) 696 : cluster 3 pg 5.e is active+undersized+degraded+remapped+backfill_wait, acting [0,4] 2023-10-09T04:30:00.989 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:00.000727+0000 mon.a (mon.0) 697 : cluster 3 pg 5.f is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:00.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:01.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:29:59.418533+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v418: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 34832/179076 objects degraded (19.451%); 7402/179076 objects misplaced (4.133%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000148+0000 mon.a (mon.0) 662 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000185+0000 mon.a (mon.0) 663 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000196+0000 mon.a (mon.0) 664 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000217+0000 mon.a (mon.0) 665 : cluster 3 osd.5 (root=default,host=smithi153) is down 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000230+0000 mon.a (mon.0) 666 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 34832/179076 objects degraded (19.451%), 17 pgs degraded, 29 pgs undersized 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000246+0000 mon.a (mon.0) 667 : cluster 3 pg 2.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,6] 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000261+0000 mon.a (mon.0) 668 : cluster 3 pg 2.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000276+0000 mon.a (mon.0) 669 : cluster 3 pg 3.2 is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:01.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000289+0000 mon.a (mon.0) 670 : cluster 3 pg 3.4 is stuck undersized for 91s, current state active+undersized, last acting [1,2] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000302+0000 mon.a (mon.0) 671 : cluster 3 pg 3.5 is stuck undersized for 67s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000313+0000 mon.a (mon.0) 672 : cluster 3 pg 3.c is stuck undersized for 67s, current state active+undersized, last acting [6,3] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000325+0000 mon.a (mon.0) 673 : cluster 3 pg 3.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,6] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000337+0000 mon.a (mon.0) 674 : cluster 3 pg 3.10 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000363+0000 mon.a (mon.0) 675 : cluster 3 pg 3.16 is stuck undersized for 91s, current state active+undersized, last acting [7,1] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000377+0000 mon.a (mon.0) 676 : cluster 3 pg 3.17 is stuck undersized for 91s, current state active+undersized, last acting [0,3] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000391+0000 mon.a (mon.0) 677 : cluster 3 pg 3.1c is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,1] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000404+0000 mon.a (mon.0) 678 : cluster 3 pg 3.1d is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000417+0000 mon.a (mon.0) 679 : cluster 3 pg 3.1f is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,2] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000431+0000 mon.a (mon.0) 680 : cluster 3 pg 4.1 is stuck undersized for 91s, current state active+undersized, last acting [4,6] 2023-10-09T04:30:01.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000443+0000 mon.a (mon.0) 681 : cluster 3 pg 4.2 is stuck undersized for 91s, current state active+undersized, last acting [1,4] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000458+0000 mon.a (mon.0) 682 : cluster 3 pg 4.3 is stuck undersized for 91s, current state active+undersized, last acting [0,7] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000472+0000 mon.a (mon.0) 683 : cluster 3 pg 4.7 is stuck undersized for 91s, current state active+undersized, last acting [1,0] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000486+0000 mon.a (mon.0) 684 : cluster 3 pg 4.8 is stuck undersized for 91s, current state active+undersized, last acting [7,6] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000500+0000 mon.a (mon.0) 685 : cluster 3 pg 4.15 is stuck undersized for 91s, current state active+undersized, last acting [7,3] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000514+0000 mon.a (mon.0) 686 : cluster 3 pg 4.1c is stuck undersized for 91s, current state active+undersized, last acting [2,3] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000528+0000 mon.a (mon.0) 687 : cluster 3 pg 4.1f is stuck undersized for 91s, current state active+undersized, last acting [6,1] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000541+0000 mon.a (mon.0) 688 : cluster 3 pg 5.2 is stuck undersized for 91s, current state active+undersized+degraded, last acting [6,0] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000563+0000 mon.a (mon.0) 689 : cluster 3 pg 5.3 is stuck undersized for 91s, current state active+undersized+degraded, last acting [0,6] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000604+0000 mon.a (mon.0) 690 : cluster 3 pg 5.4 is stuck undersized for 91s, current state active+undersized+degraded, last acting [7,2] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000621+0000 mon.a (mon.0) 691 : cluster 3 pg 5.5 is active+recovering+undersized+degraded+remapped, acting [0,1] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000636+0000 mon.a (mon.0) 692 : cluster 3 pg 5.6 is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000667+0000 mon.a (mon.0) 693 : cluster 3 pg 5.7 is stuck undersized for 91s, current state active+undersized+degraded, last acting [1,0] 2023-10-09T04:30:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000683+0000 mon.a (mon.0) 694 : cluster 3 pg 5.b is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,0] 2023-10-09T04:30:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000698+0000 mon.a (mon.0) 695 : cluster 3 pg 5.d is stuck undersized for 91s, current state active+undersized+degraded, last acting [2,7] 2023-10-09T04:30:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000713+0000 mon.a (mon.0) 696 : cluster 3 pg 5.e is active+undersized+degraded+remapped+backfill_wait, acting [0,4] 2023-10-09T04:30:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:00.000727+0000 mon.a (mon.0) 697 : cluster 3 pg 5.f is stuck undersized for 91s, current state active+undersized+degraded, last acting [4,6] 2023-10-09T04:30:01.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:01.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:01.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:01.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:01.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:01.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:01.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:01.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:01.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:01.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:01.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:01.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:01.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:01.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:01.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:02.024 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:02.024 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:02.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:02.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:02.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:02.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:02.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:02.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:02.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:02.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:02.583 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:02.687 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.714 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:02.818 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.825 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:02 smithi153 bash[59565]: cluster 2023-10-09T04:30:01.419364+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 34173/175827 objects degraded (19.436%); 7402/175827 objects misplaced (4.210%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:02.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:02.943 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:02.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:02 smithi082 bash[77063]: cluster 2023-10-09T04:30:01.419364+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 34173/175827 objects degraded (19.436%); 7402/175827 objects misplaced (4.210%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:02.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:02 smithi082 bash[79733]: cluster 2023-10-09T04:30:01.419364+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v419: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 34173/175827 objects degraded (19.436%); 7402/175827 objects misplaced (4.210%); 933 KiB/s, 14 objects/s recovering 2023-10-09T04:30:03.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:03.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:03.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:03.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:03.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:03.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:03.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:03.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:03.511 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:03.563 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:30:03.597 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:30:03.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:03.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:03.639 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:30:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:03.675 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:30:03.713 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:30:03.741 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:30:03.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:03.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:03.773 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.774 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:03.779 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 5 pgp_num to 16 2023-10-09T04:30:03.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:03.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:03.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:03.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:04.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:04.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:04.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:03 smithi082 bash[77063]: audit 2023-10-09T04:30:03.256113+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.82:0/2568364428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:03 smithi082 bash[77063]: audit 2023-10-09T04:30:03.256996+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.037 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:03 smithi082 bash[77063]: audit 2023-10-09T04:30:03.529868+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.82:0/3353666174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:04.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:04.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:03 smithi153 bash[59565]: audit 2023-10-09T04:30:03.256113+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.82:0/2568364428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:03 smithi153 bash[59565]: audit 2023-10-09T04:30:03.256996+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:03 smithi153 bash[59565]: audit 2023-10-09T04:30:03.529868+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.82:0/3353666174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:04.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:04.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:04.293 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:04.299 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:03.256113+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.82:0/2568364428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.299 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:03.256996+0000 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:30:04.299 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:03.529868+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.82:0/3353666174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:04.401 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:04.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:04.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:04.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:04.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:30:04.783 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:30:04.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:04.835 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:04.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: cluster 2023-10-09T04:30:03.420248+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 33195/170448 objects degraded (19.475%); 6943/170448 objects misplaced (4.073%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:04.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: cluster 2023-10-09T04:30:03.771088+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T04:30:04.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: audit 2023-10-09T04:30:03.771781+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:30:04.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: cluster 2023-10-09T04:30:03.771837+0000 mon.a (mon.0) 700 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: audit 2023-10-09T04:30:04.404653+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.82:0/635049891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: audit 2023-10-09T04:30:04.405321+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.457445+0000 mon.a (mon.0) 702 : cluster 3 Health check update: Degraded data redundancy: 33195/170448 objects degraded (19.475%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: cluster 2023-10-09T04:30:03.420248+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 33195/170448 objects degraded (19.475%); 6943/170448 objects misplaced (4.073%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: cluster 2023-10-09T04:30:03.771088+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:03.771781+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: cluster 2023-10-09T04:30:03.771837+0000 mon.a (mon.0) 700 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:04.404653+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.82:0/635049891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: audit 2023-10-09T04:30:04.405321+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:04 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.457445+0000 mon.a (mon.0) 702 : cluster 3 Health check update: Degraded data redundancy: 33195/170448 objects degraded (19.475%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:04.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: cluster 2023-10-09T04:30:03.420248+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v420: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 33195/170448 objects degraded (19.475%); 6943/170448 objects misplaced (4.073%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: cluster 2023-10-09T04:30:03.771088+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: audit 2023-10-09T04:30:03.771781+0000 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: cluster 2023-10-09T04:30:03.771837+0000 mon.a (mon.0) 700 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: audit 2023-10-09T04:30:04.404653+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.82:0/635049891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: audit 2023-10-09T04:30:04.405321+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:04.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:04 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.457445+0000 mon.a (mon.0) 702 : cluster 3 Health check update: Degraded data redundancy: 33195/170448 objects degraded (19.475%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:04.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:04.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:05.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:05.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:05.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:05.373 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:05.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:05.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:05.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:05.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:05.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:05.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:05.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:06.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:06.020 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.769295+0000 mon.a (mon.0) 703 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:06.020 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.773023+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: audit 2023-10-09T04:30:04.774820+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.774915+0000 mon.a (mon.0) 705 : cluster 0 osdmap e126: 8 total, 7 up, 8 in 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.785215+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[77063]: cluster 2023-10-09T04:30:04.799660+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.769295+0000 mon.a (mon.0) 703 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.773023+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: audit 2023-10-09T04:30:04.774820+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.774915+0000 mon.a (mon.0) 705 : cluster 0 osdmap e126: 8 total, 7 up, 8 in 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.785215+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.021 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:05 smithi082 bash[79733]: cluster 2023-10-09T04:30:04.799660+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:06.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.769295+0000 mon.a (mon.0) 703 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.773023+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: audit 2023-10-09T04:30:04.774820+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.774915+0000 mon.a (mon.0) 705 : cluster 0 osdmap e126: 8 total, 7 up, 8 in 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.785215+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.147 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:05 smithi153 bash[59565]: cluster 2023-10-09T04:30:04.799660+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T04:30:06.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:06.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:06.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:06.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:06.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:06.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:06.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:06.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:06 smithi082 bash[77063]: cluster 2023-10-09T04:30:05.420928+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 32310/165075 objects degraded (19.573%); 6943/165075 objects misplaced (4.206%); 720 KiB/s, 11 objects/s recovering 2023-10-09T04:30:06.835 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:06 smithi082 bash[79733]: cluster 2023-10-09T04:30:05.420928+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 32310/165075 objects degraded (19.573%); 6943/165075 objects misplaced (4.206%); 720 KiB/s, 11 objects/s recovering 2023-10-09T04:30:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:06.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:30:06.954 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:30:06.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:07.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:07.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:07.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:06 smithi153 bash[59565]: cluster 2023-10-09T04:30:05.420928+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v423: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 32310/165075 objects degraded (19.573%); 6943/165075 objects misplaced (4.206%); 720 KiB/s, 11 objects/s recovering 2023-10-09T04:30:07.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:07.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:07.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:07.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:07.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:07.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:07.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:07.573 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:07.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:07.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:07.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:07.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:07.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:07.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:07.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:08.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:08.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:08.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:08.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:08.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:08.378 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:08.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:08.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:08.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:08.612 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:08.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:08.753 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:08.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:08.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:08.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:08.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:08.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:08.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:30:09.014 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:08 smithi082 bash[77063]: cluster 2023-10-09T04:30:07.421749+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 32030/163026 objects degraded (19.647%); 6483/163026 objects misplaced (3.977%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:09.014 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:08 smithi082 bash[79733]: cluster 2023-10-09T04:30:07.421749+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 32030/163026 objects degraded (19.647%); 6483/163026 objects misplaced (3.977%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:09.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:09.079 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:09.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:09.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:08 smithi153 bash[59565]: cluster 2023-10-09T04:30:07.421749+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v424: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 32030/163026 objects degraded (19.647%); 6483/163026 objects misplaced (3.977%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:09.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:09.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:09.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:09.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:09.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:09.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:09.574 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:09.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:09.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:09.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:09] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:30:09.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:09.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:09.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:09.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:09.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:10.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.018 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:09 smithi082 bash[77063]: cluster 2023-10-09T04:30:09.460571+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 32030/163026 objects degraded (19.647%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:10.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:09 smithi082 bash[79733]: cluster 2023-10-09T04:30:09.460571+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 32030/163026 objects degraded (19.647%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:10.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:10.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:09 smithi153 bash[59565]: cluster 2023-10-09T04:30:09.460571+0000 mon.a (mon.0) 706 : cluster 3 Health check update: Degraded data redundancy: 32030/163026 objects degraded (19.647%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:10.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:10.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:10.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:10.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:10.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:10.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:10.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:10.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:10.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.868 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:10.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:10.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:10 smithi153 bash[59565]: cluster 2023-10-09T04:30:09.422430+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 31131/157599 objects degraded (19.753%); 6483/157599 objects misplaced (4.114%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:10.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:10 smithi082 bash[77063]: cluster 2023-10-09T04:30:09.422430+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 31131/157599 objects degraded (19.753%); 6483/157599 objects misplaced (4.114%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:10.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:10 smithi082 bash[79733]: cluster 2023-10-09T04:30:09.422430+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v425: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 31131/157599 objects degraded (19.753%); 6483/157599 objects misplaced (4.114%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:30:11.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:11.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:11.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:11.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:11.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:11.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:11.417 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:11.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:11.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:11.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:11.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:11.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:11.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:11.802 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:11.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:11.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:11.924 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:30:11.924 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:30:11.924 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:30:11.924 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:30:11.924 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:30:11.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:30:11.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:12.004 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:12.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:12.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:12.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:12.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:12.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:12.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:12.632 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:12.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:12.788 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:12.805 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:12 smithi082 bash[77063]: cluster 2023-10-09T04:30:11.423290+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 30455/154200 objects degraded (19.750%); 6483/154200 objects misplaced (4.204%); 688 KiB/s, 10 objects/s recovering 2023-10-09T04:30:12.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:12.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:12.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:13.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.079 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:12 smithi082 bash[79733]: cluster 2023-10-09T04:30:11.423290+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 30455/154200 objects degraded (19.750%); 6483/154200 objects misplaced (4.204%); 688 KiB/s, 10 objects/s recovering 2023-10-09T04:30:13.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:13.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:12 smithi153 bash[59565]: cluster 2023-10-09T04:30:11.423290+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v426: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 30455/154200 objects degraded (19.750%); 6483/154200 objects misplaced (4.204%); 688 KiB/s, 10 objects/s recovering 2023-10-09T04:30:13.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:13.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:13.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:13.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:13.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:13.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:13.700 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:13.701 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:13.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:13.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:13.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:13.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:13.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:13.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:13.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:14.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:14.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:14.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:14.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:14.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:14.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:14.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:14.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:14.808 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[77063]: cluster 2023-10-09T04:30:13.423964+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 29446/148686 objects degraded (19.804%); 6030/148686 objects misplaced (4.056%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:14.809 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[77063]: audit 2023-10-09T04:30:14.260094+0000 mon.a (mon.0) 707 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:14.809 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[79733]: cluster 2023-10-09T04:30:13.423964+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 29446/148686 objects degraded (19.804%); 6030/148686 objects misplaced (4.056%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:14.809 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[79733]: audit 2023-10-09T04:30:14.260094+0000 mon.a (mon.0) 707 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:14.809 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[79733]: cluster 2023-10-09T04:30:14.466288+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 29446/148686 objects degraded (19.804%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:14.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:14.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:14.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:14.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:15.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:15.068 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:14 smithi082 bash[77063]: cluster 2023-10-09T04:30:14.466288+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 29446/148686 objects degraded (19.804%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:15.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:15.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:14 smithi153 bash[59565]: cluster 2023-10-09T04:30:13.423964+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v427: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 29446/148686 objects degraded (19.804%); 6030/148686 objects misplaced (4.056%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:15.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:14 smithi153 bash[59565]: audit 2023-10-09T04:30:14.260094+0000 mon.a (mon.0) 707 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:15.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:14 smithi153 bash[59565]: cluster 2023-10-09T04:30:14.466288+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 29446/148686 objects degraded (19.804%), 17 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:15.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:15.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:15.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:15.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:15.466 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:15.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.635 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:15.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:15.772 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:15.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:15.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:15.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:16.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.027 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:16.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:16.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:16.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:16.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.443 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:16.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:16.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:16.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:16.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:16.721 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:16.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:16.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:16.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:16.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:16.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:16 smithi082 bash[77063]: cluster 2023-10-09T04:30:15.424700+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 28505/143139 objects degraded (19.914%); 6030/143139 objects misplaced (4.213%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:16.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:16 smithi082 bash[79733]: cluster 2023-10-09T04:30:15.424700+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 28505/143139 objects degraded (19.914%); 6030/143139 objects misplaced (4.213%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:17.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:17.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:16 smithi153 bash[59565]: cluster 2023-10-09T04:30:15.424700+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v428: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 28505/143139 objects degraded (19.914%); 6030/143139 objects misplaced (4.213%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:17.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:26644 5.0 5:0dd33de9:::benchmark_data_smithi082_106363_object26643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:27:27.339290+0000", 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 169.804511155, 2023-10-09T04:30:17.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.049953688000000003, 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 26644 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339290+0000", 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339290+0000", 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339288+0000", 2023-10-09T04:30:17.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339317+0000", 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8623e-05 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339332+0000", 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4802e-05 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339339+0000", 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0210000000000002e-06 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339383+0000", 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4616000000000001e-05 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339527+0000", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014426400000000001 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.339676+0000", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014874699999999999 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.341272+0000", 2023-10-09T04:30:17.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001595408 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.342431+0000", 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0011591780000000001 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.388765+0000", 2023-10-09T04:30:17.154 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.046334106999999999 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.389173+0000", 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040784799999999999 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:27:27.389244+0000", 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0840999999999995e-05 2023-10-09T04:30:17.155 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.156 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.156 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.156 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.156 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.156 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:108349 5.0 5:0d861ff8:::benchmark_data_smithi082_106363_object108348:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.157 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:39.392992+0000", 2023-10-09T04:30:17.157 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 97.750809548000007, 2023-10-09T04:30:17.157 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.043471507999999999, 2023-10-09T04:30:17.157 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.157 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 108349 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.392992+0000", 2023-10-09T04:30:17.158 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.392992+0000", 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.392990+0000", 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.159 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.393017+0000", 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7372000000000002e-05 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.393037+0000", 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9313999999999998e-05 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.160 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.393047+0000", 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0729e-05 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.393110+0000", 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2718000000000005e-05 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.161 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.394301+0000", 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00119074 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.394446+0000", 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000145103 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.403463+0000", 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0090173379999999997 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.405184+0000", 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001720968 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.436257+0000", 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031073285999999999 2023-10-09T04:30:17.163 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.436372+0000", 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000114918 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.436463+0000", 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0694999999999998e-05 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.164 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:108351 5.0 5:09308b48:::benchmark_data_smithi082_106363_object108350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:39.397534+0000", 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 97.746267015000001, 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.042262873999999999, 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.165 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 108351 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397534+0000", 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397534+0000", 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.166 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397531+0000", 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397566+0000", 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4561000000000003e-05 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.167 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397591+0000", 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5264000000000001e-05 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.397604+0000", 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2935000000000001e-05 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.399122+0000", 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0015177680000000001 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.168 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.399275+0000", 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000153275 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.399435+0000", 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015960700000000001 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.169 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.403468+0000", 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040336210000000003 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.405452+0000", 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019836929999999999 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.439635+0000", 2023-10-09T04:30:17.170 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.034182664000000001 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.439752+0000", 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011749299999999999 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:39.439797+0000", 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4601999999999997e-05 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.171 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:109854 5.0 5:055aa7cb:::benchmark_data_smithi082_106363_object109853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:41.215146+0000", 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.928654789999996, 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.045006632999999997, 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.172 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 109854 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215146+0000", 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215146+0000", 2023-10-09T04:30:17.173 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215144+0000", 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215174+0000", 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0332e-05 2023-10-09T04:30:17.174 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215200+0000", 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5541000000000002e-05 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215212+0000", 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2285e-05 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215433+0000", 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000221244 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225372+0000", 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0099390109999999993 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.226487+0000", 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001114886 2023-10-09T04:30:17.176 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.240044+0000", 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013556769999999999 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.240095+0000", 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1035999999999997e-05 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.177 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260058+0000", 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019963357000000001 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260109+0000", 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0285999999999999e-05 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.178 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260153+0000", 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4323000000000002e-05 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:109859 5.0 5:06602591:::benchmark_data_smithi082_106363_object109858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:41.215229+0000", 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.928572424999999, 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.045032174000000001, 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.179 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 109859 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215229+0000", 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.180 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215229+0000", 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215228+0000", 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.181 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215252+0000", 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4596e-05 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215261+0000", 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8980000000000002e-06 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.215268+0000", 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2770000000000002e-06 2023-10-09T04:30:17.182 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.234138+0000", 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.018870082 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.234485+0000", 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00034701600000000001 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.183 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.234893+0000", 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040799500000000003 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.259982+0000", 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.025089209000000001 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.259991+0000", 2023-10-09T04:30:17.184 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2730000000000008e-06 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260186+0000", 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019503000000000001 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260227+0000", 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0419e-05 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.185 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260261+0000", 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4261000000000003e-05 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:109863 5.0 5:032bde68:::benchmark_data_smithi082_106363_object109862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:41.225118+0000", 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.918683216000005, 2023-10-09T04:30:17.186 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.055541884, 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 109863 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.187 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225118+0000", 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225118+0000", 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225115+0000", 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999971 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.188 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225155+0000", 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0181e-05 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225181+0000", 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5559e-05 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.225193+0000", 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1688999999999999e-05 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.240201+0000", 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.015007924000000001 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.240534+0000", 2023-10-09T04:30:17.190 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00033335899999999998 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.241028+0000", 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000494456 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.259996+0000", 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.018967748 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.191 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260026+0000", 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0318999999999999e-05 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280518+0000", 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.02049202 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.192 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280619+0000", 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010009900000000001 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280660+0000", 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1199999999999999e-05 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.193 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:109872 5.0 5:0b7d28fc:::benchmark_data_smithi082_106363_object109871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:41.233849+0000", 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.909952520999994, 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.046963163000000002, 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.194 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 109872 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233849+0000", 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233849+0000", 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.195 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233846+0000", 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233876+0000", 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9722000000000001e-05 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.196 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233928+0000", 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1728999999999999e-05 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.233942+0000", 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3947e-05 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.197 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.251804+0000", 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017862387 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.252139+0000", 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00033517600000000002 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.255735+0000", 2023-10-09T04:30:17.198 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0035957910000000001 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260032+0000", 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0042969419999999998 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.273988+0000", 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013956257999999999 2023-10-09T04:30:17.199 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280696+0000", 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0067075959999999997 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280772+0000", 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6380999999999999e-05 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.200 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280812+0000", 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9348999999999999e-05 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:109875 5.0 5:01f33b97:::benchmark_data_smithi082_106363_object109874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:41.237235+0000", 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.906566419000001, 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.043765571000000003, 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.201 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 109875 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237235+0000", 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.202 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237235+0000", 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237232+0000", 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237285+0000", 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2806999999999998e-05 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237310+0000", 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4683999999999999e-05 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.204 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.237322+0000", 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2116000000000001e-05 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260288+0000", 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.022965755000000001 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260434+0000", 2023-10-09T04:30:17.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014595600000000001 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.260630+0000", 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000196138 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.274128+0000", 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013498377000000001 2023-10-09T04:30:17.206 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.274298+0000", 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000170087 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280848+0000", 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065491899999999999 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.207 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.280959+0000", 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000111099 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:41.281000+0000", 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1557999999999998e-05 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.208 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:110524 5.0 5:0ed87686:::benchmark_data_smithi082_106363_object110523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.109497829000006, 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.098985219999999999, 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 110524 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.209 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034302+0000", 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034327+0000", 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5131e-05 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034345+0000", 2023-10-09T04:30:17.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7963999999999999e-05 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034354+0000", 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2499999999999995e-06 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.038086+0000", 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0037325629999999999 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.039172+0000", 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001085637 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.039582+0000", 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000409577 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045532+0000", 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0059498820000000001 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.051599+0000", 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060676209999999996 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.214 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133142+0000", 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.081543268000000002 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133244+0000", 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010136699999999999 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133288+0000", 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4650000000000001e-05 2023-10-09T04:30:17.215 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:110532 5.0 5:06d82ee9:::benchmark_data_smithi082_106363_object110531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.043867+0000", 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.099933899000007, 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.094581456999999994, 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.216 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 110532 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043867+0000", 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043867+0000", 2023-10-09T04:30:17.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043865+0000", 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043890+0000", 2023-10-09T04:30:17.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4545999999999998e-05 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043911+0000", 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1421e-05 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.043922+0000", 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0784999999999999e-05 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045554+0000", 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0016319450000000001 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045711+0000", 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000157176 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045905+0000", 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019452599999999999 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.051604+0000", 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056989689999999999 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.053628+0000", 2023-10-09T04:30:17.221 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0020238560000000001 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138350+0000", 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.084721243000000002 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138407+0000", 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7578999999999999e-05 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138449+0000", 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1511999999999998e-05 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:110534 5.0 5:07f06b66:::benchmark_data_smithi082_106363_object110533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.045785+0000", 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 95.098016052999995, 2023-10-09T04:30:17.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.092834211999999999, 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 110534 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045785+0000", 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045785+0000", 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045784+0000", 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:17.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045806+0000", 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1807e-05 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045817+0000", 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1707e-05 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045833+0000", 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6235999999999998e-05 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.048635+0000", 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028015150000000001 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.048772+0000", 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000136594 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.048937+0000", 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016558799999999999 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.051741+0000", 2023-10-09T04:30:17.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028035540000000002 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.053673+0000", 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019326320000000001 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138494+0000", 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.084820432000000001 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138573+0000", 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8986000000000006e-05 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.138619+0000", 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6448e-05 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.230 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:111312 5.0 5:0664770c:::benchmark_data_smithi082_106363_object111311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.933790+0000", 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 94.210011550000004, 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.042530893, 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.231 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 111312 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933790+0000", 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933790+0000", 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933787+0000", 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933816+0000", 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8302000000000001e-05 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933836+0000", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0196000000000001e-05 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933846+0000", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7990000000000005e-06 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.933966+0000", 2023-10-09T04:30:17.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000120004 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934089+0000", 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012384000000000001 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934221+0000", 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013124299999999999 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.945359+0000", 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011138441000000001 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.945535+0000", 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000176176 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.976046+0000", 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.030511099999999999 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.976139+0000", 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2744000000000005e-05 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.976320+0000", 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018129299999999999 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.237 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:111314 5.0 5:057b5333:::benchmark_data_smithi082_106363_object111313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.934732+0000", 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 94.209069337000003, 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.042690587000000002, 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.238 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 111314 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934732+0000", 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.239 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934732+0000", 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934731+0000", 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934749+0000", 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8249000000000001e-05 2023-10-09T04:30:17.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934759+0000", 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0203999999999999e-05 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.934765+0000", 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9989999999999997e-06 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.937663+0000", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028977410000000001 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.938059+0000", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00039589999999999997 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.938611+0000", 2023-10-09T04:30:17.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000552463 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.945403+0000", 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.006791783 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.945803+0000", 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00039946900000000001 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.976956+0000", 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031152941 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.977363+0000", 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040721900000000001 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.977422+0000", 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9543000000000002e-05 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:115857 5.0 5:08a7e890:::benchmark_data_smithi082_106363_object1626:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:47.658894+0000", 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 89.484907346, 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.068598196, 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.245 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 115857 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658894+0000", 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658894+0000", 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658893+0000", 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658898+0000", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3140000000000003e-06 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658907+0000", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6209999999999999e-06 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658915+0000", 2023-10-09T04:30:17.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0679999999999999e-06 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.658958+0000", 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3645e-05 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.659030+0000", 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1931e-05 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.659090+0000", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0321000000000001e-05 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.661878+0000", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002787121 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.662457+0000", 2023-10-09T04:30:17.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00057977199999999999 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.727387+0000", 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.064930031999999999 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.727451+0000", 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4040000000000003e-05 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.251 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:47.727492+0000", 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0528999999999998e-05 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:117319 5.0 5:0e5629f3:::benchmark_data_smithi082_106363_object3088:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:49.185230+0000", 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 87.958570757999993, 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.043430578999999997, 2023-10-09T04:30:17.252 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 117319 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185230+0000", 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185230+0000", 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185229+0000", 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185243+0000", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4591e-05 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185252+0000", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0410000000000003e-06 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.185259+0000", 2023-10-09T04:30:17.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4130000000000003e-06 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.188395+0000", 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0031359130000000002 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.192877+0000", 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0044823609999999998 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.192996+0000", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011875900000000001 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.197333+0000", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0043372289999999997 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.202237+0000", 2023-10-09T04:30:17.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049042620000000004 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.228552+0000", 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.026314424999999999 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.228618+0000", 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6223999999999995e-05 2023-10-09T04:30:17.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.228661+0000", 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2981000000000001e-05 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:117560 5.0 5:0f3503a6:::benchmark_data_smithi082_106363_object3329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:49.448248+0000", 2023-10-09T04:30:17.259 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 87.695553200999996, 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.041150407, 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 117560 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.260 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448248+0000", 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448248+0000", 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448246+0000", 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:17.261 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448253+0000", 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3729999999999998e-06 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448270+0000", 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6855999999999999e-05 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.262 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.448279+0000", 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9320000000000002e-06 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.451104+0000", 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028254140000000001 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.263 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.451228+0000", 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000123719 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.451333+0000", 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000105746 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.459700+0000", 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0083665040000000003 2023-10-09T04:30:17.264 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.460217+0000", 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000516966 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.489110+0000", 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.028892678000000001 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.265 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.489259+0000", 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000149331 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.489398+0000", 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000139314 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.266 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:117800 5.0 5:0ccc1ccf:::benchmark_data_smithi082_106363_object3569:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:49.711649+0000", 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 87.432152216999995, 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.045775631999999997, 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 117800 2023-10-09T04:30:17.267 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711649+0000", 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711649+0000", 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.268 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711647+0000", 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711657+0000", 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0200999999999999e-05 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.269 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711679+0000", 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2132000000000001e-05 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711690+0000", 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0706999999999999e-05 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.711857+0000", 2023-10-09T04:30:17.270 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016678999999999999 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.712278+0000", 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00042101299999999998 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.712692+0000", 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00041456499999999999 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.271 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.717393+0000", 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0047009870000000002 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.719341+0000", 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019475899999999999 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.757326+0000", 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.037985038999999998 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.757399+0000", 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3107000000000003e-05 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:49.757424+0000", 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5381e-05 2023-10-09T04:30:17.273 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:118213 5.0 5:0b9fcbb9:::benchmark_data_smithi082_106363_object3982:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:50.164458+0000", 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 86.979343067000002, 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.042462861999999997, 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:30:17.274 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 118213 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164458+0000", 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.275 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164458+0000", 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164456+0000", 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164466+0000", 2023-10-09T04:30:17.276 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6490000000000003e-06 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164489+0000", 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3153e-05 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164500+0000", 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1420000000000001e-05 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.164643+0000", 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000143134 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.165088+0000", 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00044438000000000002 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.165488+0000", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040060900000000001 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.170280+0000", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0047919169999999997 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.170507+0000", 2023-10-09T04:30:17.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000226531 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.206842+0000", 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.036334762 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.206885+0000", 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3563999999999998e-05 2023-10-09T04:30:17.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:50.206921+0000", 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5722000000000001e-05 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:136668 5.a e119/106 5:568e0cba:::benchmark_data_smithi082_106363_object22437:head v 119'8488, mlcod=119'8486)", 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:11.618751+0000", 2023-10-09T04:30:17.281 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.525050249000003, 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.075584130999999999, 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.618751+0000", 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.282 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.618751+0000", 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.618750+0000", 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.618762+0000", 2023-10-09T04:30:17.283 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2055000000000001e-05 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.681524+0000", 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.062761835000000002 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.681555+0000", 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1025999999999997e-05 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.681686+0000", 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013166499999999999 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.681724+0000", 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7523e-05 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.285 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.694305+0000", 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012581126999999999 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.694335+0000", 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9961e-05 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.286 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34794.0:136681 5.1 e120/106 5:8e72fbd5:::benchmark_data_smithi082_106363_object22450:head v 120'8592, mlcod=119'8590)", 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:11.643597+0000", 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.500203807999995, 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.041276729999999998, 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.643597+0000", 2023-10-09T04:30:17.287 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.643597+0000", 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.643597+0000", 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.643606+0000", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7539999999999995e-06 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.682025+0000", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.038419492999999999 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.682034+0000", 2023-10-09T04:30:17.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8480000000000007e-06 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.682062+0000", 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7690000000000001e-05 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.682080+0000", 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8688000000000001e-05 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.684840+0000", 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027596230000000001 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:11.684874+0000", 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4025e-05 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.291 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.292 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:30:17.292 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:30:17.292 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:30:17.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.322 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:17.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:17.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:17.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:17.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:17.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:17.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:17.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:17.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:17.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.987 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:17.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:18.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:18.125 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:18.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:18.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:18.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:18.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:18.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:18.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:18.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:18.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:18.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:18.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:18.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:18.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:18.901 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:18 smithi082 bash[77063]: cluster 2023-10-09T04:30:17.425620+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v429: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 28164/141048 objects degraded (19.968%); 5577/141048 objects misplaced (3.954%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:18.901 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:18 smithi082 bash[79733]: cluster 2023-10-09T04:30:17.425620+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v429: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 28164/141048 objects degraded (19.968%); 5577/141048 objects misplaced (3.954%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:18.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:18.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:19.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:19.035 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:18 smithi153 bash[59565]: cluster 2023-10-09T04:30:17.425620+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v429: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 28164/141048 objects degraded (19.968%); 5577/141048 objects misplaced (3.954%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:19.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:19.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:19.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:19.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:19.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:19.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:19.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:19.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:19.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:19.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:19.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:19.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:19.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:19.858 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:19.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:19.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:19.972 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:19 smithi082 bash[77063]: cluster 2023-10-09T04:30:19.467831+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 28164/141048 objects degraded (19.968%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:19.972 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:19 smithi082 bash[79733]: cluster 2023-10-09T04:30:19.467831+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 28164/141048 objects degraded (19.968%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:19.973 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:19] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-09T04:30:19.996 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:20.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.127 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:20.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:19 smithi153 bash[59565]: cluster 2023-10-09T04:30:19.467831+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 28164/141048 objects degraded (19.968%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:20.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:20.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:20.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:20.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:20.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:20.680 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:20.786 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:20.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:20.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:20.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:20.996 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:30:20.996 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "" 2023-10-09T04:30:20.996 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:30:21.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:21.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:20 smithi082 bash[77063]: cluster 2023-10-09T04:30:19.426703+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v430: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 27253/135627 objects degraded (20.094%); 5577/135627 objects misplaced (4.112%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:20 smithi082 bash[79733]: cluster 2023-10-09T04:30:19.426703+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v430: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 27253/135627 objects degraded (20.094%); 5577/135627 objects misplaced (4.112%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:21.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:21.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:21.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:20 smithi153 bash[59565]: cluster 2023-10-09T04:30:19.426703+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v430: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 27253/135627 objects degraded (20.094%); 5577/135627 objects misplaced (4.112%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:21.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:21.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:21.321 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:21.454 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.482 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:21.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:21.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:30:21.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:21.724 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.748 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:21.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:21.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:21 smithi082 bash[77063]: cluster 2023-10-09T04:30:21.427660+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 26577/132096 objects degraded (20.119%); 5577/132096 objects misplaced (4.222%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:21.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:21 smithi082 bash[79733]: cluster 2023-10-09T04:30:21.427660+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 26577/132096 objects degraded (20.119%); 5577/132096 objects misplaced (4.222%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:21.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:22.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:22.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:22.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:21 smithi153 bash[59565]: cluster 2023-10-09T04:30:21.427660+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v431: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 26577/132096 objects degraded (20.119%); 5577/132096 objects misplaced (4.222%); 954 KiB/s, 14 objects/s recovering 2023-10-09T04:30:22.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:22.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:22.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:22.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:22.653 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:22.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:22.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:22.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:22.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:22.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:22.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:23.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:23.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:23.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:23.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:23.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:23.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:23.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:23.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:23.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:23.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:23.677 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:30:23.714 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:30:23.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:23.762 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:30:23.776 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:23.806 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:30:23.852 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:30:23.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:23.890 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:30:23.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:23.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:23.931 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:30:23.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:30:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:30:24.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:24.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:24.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:24.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:24.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:24.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:24.505 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:30:24.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:24.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:24 smithi153 bash[59565]: cluster 2023-10-09T04:30:23.428540+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v432: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 25587/126372 objects degraded (20.247%); 5067/126372 objects misplaced (4.010%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:24.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:24 smithi153 bash[59565]: audit 2023-10-09T04:30:23.613819+0000 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.82:0/564968295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:24.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:24 smithi153 bash[59565]: audit 2023-10-09T04:30:24.178249+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.82:0/1477884542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:24 smithi153 bash[59565]: audit 2023-10-09T04:30:24.179256+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:24 smithi153 bash[59565]: cluster 2023-10-09T04:30:24.473158+0000 mon.a (mon.0) 712 : cluster 3 Health check update: Degraded data redundancy: 25587/126372 objects degraded (20.247%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:24.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:24.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:24.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:24.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:24.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[77063]: cluster 2023-10-09T04:30:23.428540+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v432: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 25587/126372 objects degraded (20.247%); 5067/126372 objects misplaced (4.010%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[77063]: audit 2023-10-09T04:30:23.613819+0000 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.82:0/564968295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[77063]: audit 2023-10-09T04:30:24.178249+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.82:0/1477884542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[77063]: audit 2023-10-09T04:30:24.179256+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[77063]: cluster 2023-10-09T04:30:24.473158+0000 mon.a (mon.0) 712 : cluster 3 Health check update: Degraded data redundancy: 25587/126372 objects degraded (20.247%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[79733]: cluster 2023-10-09T04:30:23.428540+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v432: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 25587/126372 objects degraded (20.247%); 5067/126372 objects misplaced (4.010%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[79733]: audit 2023-10-09T04:30:23.613819+0000 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.82:0/564968295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[79733]: audit 2023-10-09T04:30:24.178249+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.82:0/1477884542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[79733]: audit 2023-10-09T04:30:24.179256+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:24 smithi082 bash[79733]: cluster 2023-10-09T04:30:24.473158+0000 mon.a (mon.0) 712 : cluster 3 Health check update: Degraded data redundancy: 25587/126372 objects degraded (20.247%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:24.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:25.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:25.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:25.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:25.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:25.266 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:25.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:25.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:25.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:25.533 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:25.638 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:25.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:25.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:30:25.716 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[77063]: cluster 2023-10-09T04:30:24.483845+0000 mon.a (mon.0) 713 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[77063]: cluster 2023-10-09T04:30:24.489662+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[77063]: audit 2023-10-09T04:30:24.495981+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[77063]: cluster 2023-10-09T04:30:24.496231+0000 mon.a (mon.0) 715 : cluster 0 osdmap e127: 8 total, 7 up, 8 in 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[77063]: cluster 2023-10-09T04:30:24.501886+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:25.717 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[79733]: cluster 2023-10-09T04:30:24.483845+0000 mon.a (mon.0) 713 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:25.718 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[79733]: cluster 2023-10-09T04:30:24.489662+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:25.718 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[79733]: audit 2023-10-09T04:30:24.495981+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:25.718 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[79733]: cluster 2023-10-09T04:30:24.496231+0000 mon.a (mon.0) 715 : cluster 0 osdmap e127: 8 total, 7 up, 8 in 2023-10-09T04:30:25.718 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:25 smithi082 bash[79733]: cluster 2023-10-09T04:30:24.501886+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:25.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:25.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:25.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:25.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:25.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:25.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:25 smithi153 bash[59565]: cluster 2023-10-09T04:30:24.483845+0000 mon.a (mon.0) 713 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:25.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:25 smithi153 bash[59565]: cluster 2023-10-09T04:30:24.489662+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:25.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:25 smithi153 bash[59565]: audit 2023-10-09T04:30:24.495981+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:25.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:25 smithi153 bash[59565]: cluster 2023-10-09T04:30:24.496231+0000 mon.a (mon.0) 715 : cluster 0 osdmap e127: 8 total, 7 up, 8 in 2023-10-09T04:30:25.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:25 smithi153 bash[59565]: cluster 2023-10-09T04:30:24.501886+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T04:30:26.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:26.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:26.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:26.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:26.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:26.319 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:26.422 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:26.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:26.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:26.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:26.566 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:26 smithi082 bash[77063]: cluster 2023-10-09T04:30:25.429778+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v434: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24661/120819 objects degraded (20.412%); 5067/120819 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:26.567 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:26 smithi082 bash[79733]: cluster 2023-10-09T04:30:25.429778+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v434: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24661/120819 objects degraded (20.412%); 5067/120819 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:26.583 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:26.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:26.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:26.786 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:30:26.786 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:30:26.786 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T04:30:26.787 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:30:26.840 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:26.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:26.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:26.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:26 smithi153 bash[59565]: cluster 2023-10-09T04:30:25.429778+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v434: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24661/120819 objects degraded (20.412%); 5067/120819 objects misplaced (4.194%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:26.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:27.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:27.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:27.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:27.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.277 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:27.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:27.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:27.524 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.556 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:27.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:27.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:27.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:30:27.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:27.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:27.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:27.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:27.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:28.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:28.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:28.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:28.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:28.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:28.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:30:28.333 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:28.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:28.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:28.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:28.521 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:28 smithi153 bash[59565]: cluster 2023-10-09T04:30:27.430625+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v435: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 24331/118755 objects degraded (20.488%); 4615/118755 objects misplaced (3.886%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:28.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:28.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:28.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:28 smithi082 bash[77063]: cluster 2023-10-09T04:30:27.430625+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v435: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 24331/118755 objects degraded (20.488%); 4615/118755 objects misplaced (3.886%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:28.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:28 smithi082 bash[79733]: cluster 2023-10-09T04:30:27.430625+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v435: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 24331/118755 objects degraded (20.488%); 4615/118755 objects misplaced (3.886%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:28.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:28.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:28.887 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:28.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:28.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:29.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:29.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:29.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:29.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:29.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:29.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:29.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:29.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:29.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:29.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:29 smithi082 bash[77063]: audit 2023-10-09T04:30:29.260405+0000 mon.a (mon.0) 716 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:29.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:29 smithi082 bash[77063]: cluster 2023-10-09T04:30:29.474632+0000 mon.a (mon.0) 717 : cluster 3 Health check update: Degraded data redundancy: 24331/118755 objects degraded (20.488%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:29.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:29] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-09T04:30:29.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:29 smithi082 bash[79733]: audit 2023-10-09T04:30:29.260405+0000 mon.a (mon.0) 716 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:29.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:29 smithi082 bash[79733]: cluster 2023-10-09T04:30:29.474632+0000 mon.a (mon.0) 717 : cluster 3 Health check update: Degraded data redundancy: 24331/118755 objects degraded (20.488%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:29.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:29.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:29.828 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:29 smithi153 bash[59565]: audit 2023-10-09T04:30:29.260405+0000 mon.a (mon.0) 716 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:29.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:29 smithi153 bash[59565]: cluster 2023-10-09T04:30:29.474632+0000 mon.a (mon.0) 717 : cluster 3 Health check update: Degraded data redundancy: 24331/118755 objects degraded (20.488%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:29.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:29.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:29.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:29.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:30.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:30.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:30.104 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:30.104 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:30.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:30.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:30.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:30.279 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:30.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:30.424 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:30.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.537 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:30 smithi082 bash[77063]: cluster 2023-10-09T04:30:29.431483+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v436: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 23370/113187 objects degraded (20.647%); 4615/113187 objects misplaced (4.077%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:30.554 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:30 smithi153 bash[59565]: cluster 2023-10-09T04:30:29.431483+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v436: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 23370/113187 objects degraded (20.647%); 4615/113187 objects misplaced (4.077%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:30.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:30.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:30.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:30.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:30 smithi082 bash[79733]: cluster 2023-10-09T04:30:29.431483+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v436: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 23370/113187 objects degraded (20.647%); 4615/113187 objects misplaced (4.077%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:30.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:30:30.950 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:30.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:31.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:31.097 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:31.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:31.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:31.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:31.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:31.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:31.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:31.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:31.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:31.633 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:31.633 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:31.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:31.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:31.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:31.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:31.907 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:31.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:31.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:32.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:32.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:32.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:32.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:32.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:32.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:32.532 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:32.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:32.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:32.644 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:32.666 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.673 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:32 smithi153 bash[59565]: cluster 2023-10-09T04:30:31.432609+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v437: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 22700/109809 objects degraded (20.672%); 4615/109809 objects misplaced (4.203%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:32.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:32.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.805 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:32 smithi082 bash[77063]: cluster 2023-10-09T04:30:31.432609+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v437: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 22700/109809 objects degraded (20.672%); 4615/109809 objects misplaced (4.203%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:32.805 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:32 smithi082 bash[79733]: cluster 2023-10-09T04:30:31.432609+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v437: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 22700/109809 objects degraded (20.672%); 4615/109809 objects misplaced (4.203%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:32.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:32.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:32.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:33.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:33.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:33.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:33.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:33.224 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:33.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.358 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:33.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:33.590 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:33.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:33.735 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:33.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:33.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:33.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:34.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:34.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:34.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:34.132 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:34.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:34.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:34.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:34.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:34.673 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:34.682 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:34 smithi082 bash[77063]: cluster 2023-10-09T04:30:33.433474+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v438: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 21746/104487 objects degraded (20.812%); 4143/104487 objects misplaced (3.965%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:34.682 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:34 smithi082 bash[77063]: cluster 2023-10-09T04:30:34.475957+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 21746/104487 objects degraded (20.812%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:34.682 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:34 smithi082 bash[79733]: cluster 2023-10-09T04:30:33.433474+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v438: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 21746/104487 objects degraded (20.812%); 4143/104487 objects misplaced (3.965%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:34.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:34 smithi082 bash[79733]: cluster 2023-10-09T04:30:34.475957+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 21746/104487 objects degraded (20.812%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:34.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:34.811 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:34.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:34 smithi153 bash[59565]: cluster 2023-10-09T04:30:33.433474+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v438: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 21746/104487 objects degraded (20.812%); 4143/104487 objects misplaced (3.965%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:34.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:34 smithi153 bash[59565]: cluster 2023-10-09T04:30:34.475957+0000 mon.a (mon.0) 718 : cluster 3 Health check update: Degraded data redundancy: 21746/104487 objects degraded (20.812%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:34.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:34.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:34.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:34.977 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:34.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:34.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:35.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:35.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:35.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:35.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:35.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:35.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:35.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:35.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:35.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:35.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.828 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:35.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:35.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:36.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:36.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:36.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:36.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:36.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:36.379 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:36.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:36.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:36.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:36.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:36 smithi082 bash[77063]: cluster 2023-10-09T04:30:35.434405+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v439: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:36.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:36 smithi082 bash[79733]: cluster 2023-10-09T04:30:35.434405+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v439: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:36.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:36.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.904 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:36.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:36.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:36 smithi153 bash[59565]: cluster 2023-10-09T04:30:35.434405+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v439: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:37.007 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:37.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:37.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:37.176 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:37.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:37.280 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:37.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:37.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:37.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:37.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:37.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:37.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:37.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:37.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:37.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:37.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:37.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:37.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:38.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:38.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:38.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:38.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:38.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:38.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:38.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:38.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:38.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:38 smithi153 bash[59565]: cluster 2023-10-09T04:30:37.435197+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v440: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:38.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:38.710 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:38 smithi082 bash[77063]: cluster 2023-10-09T04:30:37.435197+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v440: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:38.710 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:38 smithi082 bash[79733]: cluster 2023-10-09T04:30:37.435197+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v440: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20850/99081 objects degraded (21.043%); 4143/99081 objects misplaced (4.181%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:30:38.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:38.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:38.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:38.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:39.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:39.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:39.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:39.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:39.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:39.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:39.657 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.665 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:39 smithi082 bash[77063]: cluster 2023-10-09T04:30:39.477279+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 20850/99081 objects degraded (21.043%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:39.666 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:39 smithi082 bash[79733]: cluster 2023-10-09T04:30:39.477279+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 20850/99081 objects degraded (21.043%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:39.671 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:39.671 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:39.680 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:39.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.795 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:39 smithi153 bash[59565]: cluster 2023-10-09T04:30:39.477279+0000 mon.a (mon.0) 719 : cluster 3 Health check update: Degraded data redundancy: 20850/99081 objects degraded (21.043%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:39.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:39.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:39.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:39.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:39.989 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:39] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-09T04:30:40.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:40.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:40.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:40.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:40.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:40.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:40.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:40.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:40.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:40.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:40.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:30:40.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:40.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:40.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:40.724 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:40 smithi153 bash[59565]: cluster 2023-10-09T04:30:39.436128+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v441: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 19549/91356 objects degraded (21.399%); 3675/91356 objects misplaced (4.023%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:40.748 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:40.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:40 smithi082 bash[77063]: cluster 2023-10-09T04:30:39.436128+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v441: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 19549/91356 objects degraded (21.399%); 3675/91356 objects misplaced (4.023%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:40.762 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:40 smithi082 bash[79733]: cluster 2023-10-09T04:30:39.436128+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v441: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.9 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 19549/91356 objects degraded (21.399%); 3675/91356 objects misplaced (4.023%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:40.857 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:40.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:40.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:40.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:40.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:41.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:41.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:41.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:41.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:41.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:41.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:41.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:41.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:41.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:41.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:41.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:41.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:41.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:41.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:42.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:42.080 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:42.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:42.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:42.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:42.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:42.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:42.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:42.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:42.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:42 smithi082 bash[77063]: cluster 2023-10-09T04:30:41.437045+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v442: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18834/87708 objects degraded (21.474%); 3675/87708 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:42.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:42 smithi082 bash[79733]: cluster 2023-10-09T04:30:41.437045+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v442: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18834/87708 objects degraded (21.474%); 3675/87708 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:42.733 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:42.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:42.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:42.869 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:42 smithi153 bash[59565]: cluster 2023-10-09T04:30:41.437045+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v442: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18834/87708 objects degraded (21.474%); 3675/87708 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:30:42.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:42.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:42.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:43.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:43.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:43.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:43.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:43.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:43.400 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:43.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:43.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:43.562 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:43.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:43.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:43.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:43.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:43.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:43.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:43.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:44.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:44.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:44.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:44.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:44.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:44.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:44.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:44 smithi153 bash[59565]: cluster 2023-10-09T04:30:43.437956+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v443: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 17741/82008 objects degraded (21.633%); 3242/82008 objects misplaced (3.953%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:30:44.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:44 smithi153 bash[59565]: audit 2023-10-09T04:30:44.260766+0000 mon.a (mon.0) 720 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:44.596 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:44 smithi153 bash[59565]: cluster 2023-10-09T04:30:44.478373+0000 mon.a (mon.0) 721 : cluster 3 Health check update: Degraded data redundancy: 17741/82008 objects degraded (21.633%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:44.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:44.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[77063]: cluster 2023-10-09T04:30:43.437956+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v443: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 17741/82008 objects degraded (21.633%); 3242/82008 objects misplaced (3.953%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:30:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[77063]: audit 2023-10-09T04:30:44.260766+0000 mon.a (mon.0) 720 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[77063]: cluster 2023-10-09T04:30:44.478373+0000 mon.a (mon.0) 721 : cluster 3 Health check update: Degraded data redundancy: 17741/82008 objects degraded (21.633%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[79733]: cluster 2023-10-09T04:30:43.437956+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v443: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 17741/82008 objects degraded (21.633%); 3242/82008 objects misplaced (3.953%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:30:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[79733]: audit 2023-10-09T04:30:44.260766+0000 mon.a (mon.0) 720 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:44.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:44 smithi082 bash[79733]: cluster 2023-10-09T04:30:44.478373+0000 mon.a (mon.0) 721 : cluster 3 Health check update: Degraded data redundancy: 17741/82008 objects degraded (21.633%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:44.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:44.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:44.867 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:30:44.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:44.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:44.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:44.903 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:30:44.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:30:44.988 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:30:45.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.034 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:30:45.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:45.085 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:30:45.128 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:30:45.130 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:30:45.130 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:30:45.130 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.130 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.130 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.131 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.131 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:30:45.131 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.131 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:45.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:45.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:45.373 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:45.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:45.532 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:45.607 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:30:45.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[77063]: audit 2023-10-09T04:30:44.737448+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.82:0/640397046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[77063]: audit 2023-10-09T04:30:44.738116+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[77063]: audit 2023-10-09T04:30:44.813332+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.82:0/2911695951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:45.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[79733]: audit 2023-10-09T04:30:44.737448+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.82:0/640397046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[79733]: audit 2023-10-09T04:30:44.738116+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:45 smithi082 bash[79733]: audit 2023-10-09T04:30:44.813332+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.82:0/2911695951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:45.667 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:45 smithi153 bash[59565]: audit 2023-10-09T04:30:44.737448+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.82:0/640397046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:45 smithi153 bash[59565]: audit 2023-10-09T04:30:44.738116+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:45 smithi153 bash[59565]: audit 2023-10-09T04:30:44.813332+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.82:0/2911695951' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:45.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:45.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:45.826 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:45.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:45.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:46.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:46.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:46.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:46.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:46.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:46.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:46.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:46.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:46.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: audit 2023-10-09T04:30:45.104460+0000 mgr.y (mgr.34107) 492 : audit 0 from='client.44846 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:30:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: cluster 2023-10-09T04:30:45.438665+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v444: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: cluster 2023-10-09T04:30:45.579334+0000 mon.a (mon.0) 723 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: cluster 2023-10-09T04:30:45.589704+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: audit 2023-10-09T04:30:45.594816+0000 mon.a (mon.0) 724 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[79733]: cluster 2023-10-09T04:30:45.595261+0000 mon.a (mon.0) 725 : cluster 0 osdmap e128: 8 total, 7 up, 8 in 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: audit 2023-10-09T04:30:45.104460+0000 mgr.y (mgr.34107) 492 : audit 0 from='client.44846 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: cluster 2023-10-09T04:30:45.438665+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v444: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: cluster 2023-10-09T04:30:45.579334+0000 mon.a (mon.0) 723 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: cluster 2023-10-09T04:30:45.589704+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: audit 2023-10-09T04:30:45.594816+0000 mon.a (mon.0) 724 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:46 smithi082 bash[77063]: cluster 2023-10-09T04:30:45.595261+0000 mon.a (mon.0) 725 : cluster 0 osdmap e128: 8 total, 7 up, 8 in 2023-10-09T04:30:46.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:46.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: audit 2023-10-09T04:30:45.104460+0000 mgr.y (mgr.34107) 492 : audit 0 from='client.44846 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:30:46.850 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: cluster 2023-10-09T04:30:45.438665+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v444: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:30:46.850 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: cluster 2023-10-09T04:30:45.579334+0000 mon.a (mon.0) 723 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:46.850 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: cluster 2023-10-09T04:30:45.589704+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T04:30:46.850 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: audit 2023-10-09T04:30:45.594816+0000 mon.a (mon.0) 724 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:46.850 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:46 smithi153 bash[59565]: cluster 2023-10-09T04:30:45.595261+0000 mon.a (mon.0) 725 : cluster 0 osdmap e128: 8 total, 7 up, 8 in 2023-10-09T04:30:46.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:46.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:46.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:47.020 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:47.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:47.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:47.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:47.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:47.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:47.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:47.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:47.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:47.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:47.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:48.079 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":443,"stamp":"2023-10-09T04:30:43.437263+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1794381771,"num_objects":27336,"num_object_clones":0,"num_object_copies":82008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17741,"num_objects_misplaced":3242,"num_objects_unfound":0,"num_objects_dirty":27336,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":202256,"num_write_kb":7325752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104857600,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144620,"ondisk_log_size":144620,"up":236,"acting":236,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9739668,"kb_used_data":8083988,"kb_used_omap":34,"kb_used_meta":1655133,"kb_avail":740221548,"statfs":{"total":767960285184,"available":757986865152,"internally_reserved":0,"allocated":8278003712,"data_stored":8273576637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1694856917},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-607322112,"num_objects":-9267,"num_object_clones":0,"num_object_copies":-27801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4959,"num_objects_misplaced":-1373,"num_objects_unfound":0,"num_objects_dirty":-9267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9267,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":320,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005385"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796370+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:30:04.796370+0000","last_peered":"2023-10-09T04:30:04.796370+0000","last_clean":"2023-10-09T04:30:04.796370+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:30:04.796370+0000","last_undegraded":"2023-10-09T04:30:04.796370+0000","last_fullsized":"2023-10-09T04:30:04.796370+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.795689+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:30:04.795689+0000","last_peered":"2023-10-09T04:30:04.795689+0000","last_clean":"2023-10-09T04:30:04.795689+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:30:04.795689+0000","last_undegraded":"2023-10-09T04:30:04.795689+0000","last_fullsized":"2023-10-09T04:30:04.795689+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.789973+0000","last_change":"2023-10-09T04:29:12.641467+0000","last_active":"2023-10-09T04:30:04.789973+0000","last_peered":"2023-10-09T04:30:04.789973+0000","last_clean":"2023-10-09T04:30:04.789973+0000","last_became_active":"2023-10-09T04:29:12.641036+0000","last_became_peered":"2023-10-09T04:29:12.641036+0000","last_unstale":"2023-10-09T04:30:04.789973+0000","last_undegraded":"2023-10-09T04:30:04.789973+0000","last_fullsized":"2023-10-09T04:30:04.789973+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:41.624520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:22.404168+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:30:22.404168+0000","last_peered":"2023-10-09T04:30:22.404168+0000","last_clean":"2023-10-09T04:30:22.404168+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:30:22.404168+0000","last_undegraded":"2023-10-09T04:30:22.404168+0000","last_fullsized":"2023-10-09T04:30:22.404168+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.820945+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:30:04.820945+0000","last_peered":"2023-10-09T04:30:04.820945+0000","last_clean":"2023-10-09T04:30:04.820945+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:30:04.820945+0000","last_undegraded":"2023-10-09T04:30:04.820945+0000","last_fullsized":"2023-10-09T04:30:04.820945+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.797298+0000","last_change":"2023-10-09T04:28:27.618032+0000","last_active":"2023-10-09T04:30:04.797298+0000","last_peered":"2023-10-09T04:30:04.797298+0000","last_clean":"2023-10-09T04:28:19.577947+0000","last_became_active":"2023-10-09T04:28:27.618032+0000","last_became_peered":"2023-10-09T04:28:27.618032+0000","last_unstale":"2023-10-09T04:30:04.797298+0000","last_undegraded":"2023-10-09T04:30:04.797298+0000","last_fullsized":"2023-10-09T04:28:27.603354+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":84,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.805275+0000","last_change":"2023-10-09T04:28:27.599874+0000","last_active":"2023-10-09T04:30:04.805275+0000","last_peered":"2023-10-09T04:30:04.805275+0000","last_clean":"2023-10-09T04:28:19.571527+0000","last_became_active":"2023-10-09T04:28:27.599874+0000","last_became_peered":"2023-10-09T04:28:27.599874+0000","last_unstale":"2023-10-09T04:30:04.805275+0000","last_undegraded":"2023-10-09T04:30:04.805275+0000","last_fullsized":"2023-10-09T04:28:27.597863+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:45.767094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790313+0000","last_change":"2023-10-09T04:28:51.528372+0000","last_active":"2023-10-09T04:30:04.790313+0000","last_peered":"2023-10-09T04:30:04.790313+0000","last_clean":"2023-10-09T04:30:04.790313+0000","last_became_active":"2023-10-09T04:28:51.527988+0000","last_became_peered":"2023-10-09T04:28:51.527988+0000","last_unstale":"2023-10-09T04:30:04.790313+0000","last_undegraded":"2023-10-09T04:30:04.790313+0000","last_fullsized":"2023-10-09T04:30:04.790313+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.258888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.805911+0000","last_change":"2023-10-09T04:28:27.608355+0000","last_active":"2023-10-09T04:30:04.805911+0000","last_peered":"2023-10-09T04:30:04.805911+0000","last_clean":"2023-10-09T04:22:23.598572+0000","last_became_active":"2023-10-09T04:28:27.608355+0000","last_became_peered":"2023-10-09T04:28:27.608355+0000","last_unstale":"2023-10-09T04:30:04.805911+0000","last_undegraded":"2023-10-09T04:30:04.805911+0000","last_fullsized":"2023-10-09T04:28:27.604613+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806017+0000","last_change":"2023-10-09T04:28:51.533081+0000","last_active":"2023-10-09T04:30:04.806017+0000","last_peered":"2023-10-09T04:30:04.806017+0000","last_clean":"2023-10-09T04:30:04.806017+0000","last_became_active":"2023-10-09T04:28:51.532684+0000","last_became_peered":"2023-10-09T04:28:51.532684+0000","last_unstale":"2023-10-09T04:30:04.806017+0000","last_undegraded":"2023-10-09T04:30:04.806017+0000","last_fullsized":"2023-10-09T04:30:04.806017+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.808820+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:30:04.808820+0000","last_peered":"2023-10-09T04:30:04.808820+0000","last_clean":"2023-10-09T04:30:04.808820+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:30:04.808820+0000","last_undegraded":"2023-10-09T04:30:04.808820+0000","last_fullsized":"2023-10-09T04:30:04.808820+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790530+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:30:04.790530+0000","last_peered":"2023-10-09T04:30:04.790530+0000","last_clean":"2023-10-09T04:30:04.790530+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:30:04.790530+0000","last_undegraded":"2023-10-09T04:30:04.790530+0000","last_fullsized":"2023-10-09T04:30:04.790530+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":100,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.797322+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:30:04.797322+0000","last_peered":"2023-10-09T04:30:04.797322+0000","last_clean":"2023-10-09T04:30:04.797322+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:30:04.797322+0000","last_undegraded":"2023-10-09T04:30:04.797322+0000","last_fullsized":"2023-10-09T04:30:04.797322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796236+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:30:04.796236+0000","last_peered":"2023-10-09T04:30:04.796236+0000","last_clean":"2023-10-09T04:30:04.796236+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:30:04.796236+0000","last_undegraded":"2023-10-09T04:30:04.796236+0000","last_fullsized":"2023-10-09T04:30:04.796236+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805361+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:30:04.805361+0000","last_peered":"2023-10-09T04:30:04.805361+0000","last_clean":"2023-10-09T04:30:04.805361+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:30:04.805361+0000","last_undegraded":"2023-10-09T04:30:04.805361+0000","last_fullsized":"2023-10-09T04:30:04.805361+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805088+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:30:04.805088+0000","last_peered":"2023-10-09T04:30:04.805088+0000","last_clean":"2023-10-09T04:30:04.805088+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:30:04.805088+0000","last_undegraded":"2023-10-09T04:30:04.805088+0000","last_fullsized":"2023-10-09T04:30:04.805088+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804383+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:30:04.804383+0000","last_peered":"2023-10-09T04:30:04.804383+0000","last_clean":"2023-10-09T04:30:04.804383+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:30:04.804383+0000","last_undegraded":"2023-10-09T04:30:04.804383+0000","last_fullsized":"2023-10-09T04:30:04.804383+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.805561+0000","last_change":"2023-10-09T04:28:27.609484+0000","last_active":"2023-10-09T04:30:04.805561+0000","last_peered":"2023-10-09T04:30:04.805561+0000","last_clean":"2023-10-09T04:23:18.758860+0000","last_became_active":"2023-10-09T04:28:27.609484+0000","last_became_peered":"2023-10-09T04:28:27.609484+0000","last_unstale":"2023-10-09T04:30:04.805561+0000","last_undegraded":"2023-10-09T04:30:04.805561+0000","last_fullsized":"2023-10-09T04:28:27.606325+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806228+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:30:04.806228+0000","last_peered":"2023-10-09T04:30:04.806228+0000","last_clean":"2023-10-09T04:30:04.806228+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:30:04.806228+0000","last_undegraded":"2023-10-09T04:30:04.806228+0000","last_fullsized":"2023-10-09T04:30:04.806228+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806144+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:30:04.806144+0000","last_peered":"2023-10-09T04:30:04.806144+0000","last_clean":"2023-10-09T04:30:04.806144+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:30:04.806144+0000","last_undegraded":"2023-10-09T04:30:04.806144+0000","last_fullsized":"2023-10-09T04:30:04.806144+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":107,"reported_epoch":126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:04.795267+0000","last_change":"2023-10-09T04:28:27.608819+0000","last_active":"2023-10-09T04:30:04.795267+0000","last_peered":"2023-10-09T04:30:04.795267+0000","last_clean":"2023-10-09T04:28:19.577481+0000","last_became_active":"2023-10-09T04:28:27.608819+0000","last_became_peered":"2023-10-09T04:28:27.608819+0000","last_unstale":"2023-10-09T04:30:04.795267+0000","last_undegraded":"2023-10-09T04:28:27.602582+0000","last_fullsized":"2023-10-09T04:28:27.602406+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805957+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:30:04.805957+0000","last_peered":"2023-10-09T04:30:04.805957+0000","last_clean":"2023-10-09T04:30:04.805957+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:30:04.805957+0000","last_undegraded":"2023-10-09T04:30:04.805957+0000","last_fullsized":"2023-10-09T04:30:04.805957+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"127'12640","reported_seq":12669,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:40.296585+0000","last_change":"2023-10-09T04:26:55.440461+0000","last_active":"2023-10-09T04:30:40.296585+0000","last_peered":"2023-10-09T04:30:40.296585+0000","last_clean":"2023-10-09T04:30:40.296585+0000","last_became_active":"2023-10-09T04:26:55.440099+0000","last_became_peered":"2023-10-09T04:26:55.440099+0000","last_unstale":"2023-10-09T04:30:40.296585+0000","last_undegraded":"2023-10-09T04:30:40.296585+0000","last_fullsized":"2023-10-09T04:30:40.296585+0000","mapping_epoch":106,"log_start":"111'4000","ondisk_log_start":"111'4000","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8640,"log_dups_size":0,"ondisk_log_size":8640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119144448,"num_objects":1818,"num_object_clones":0,"num_object_copies":5454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12640,"num_write_kb":462656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806127+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:30:04.806127+0000","last_peered":"2023-10-09T04:30:04.806127+0000","last_clean":"2023-10-09T04:30:04.806127+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:30:04.806127+0000","last_undegraded":"2023-10-09T04:30:04.806127+0000","last_fullsized":"2023-10-09T04:30:04.806127+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.804503+0000","last_change":"2023-10-09T04:28:27.610146+0000","last_active":"2023-10-09T04:30:04.804503+0000","last_peered":"2023-10-09T04:30:04.804503+0000","last_clean":"2023-10-09T04:11:26.190073+0000","last_became_active":"2023-10-09T04:28:27.610146+0000","last_became_peered":"2023-10-09T04:28:27.610146+0000","last_unstale":"2023-10-09T04:30:04.804503+0000","last_undegraded":"2023-10-09T04:30:04.804503+0000","last_fullsized":"2023-10-09T04:28:27.604381+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"127'12634","reported_seq":12664,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:42.587607+0000","last_change":"2023-10-09T04:26:55.433159+0000","last_active":"2023-10-09T04:30:42.587607+0000","last_peered":"2023-10-09T04:30:42.587607+0000","last_clean":"2023-10-09T04:30:42.587607+0000","last_became_active":"2023-10-09T04:26:55.432784+0000","last_became_peered":"2023-10-09T04:26:55.432784+0000","last_unstale":"2023-10-09T04:30:42.587607+0000","last_undegraded":"2023-10-09T04:30:42.587607+0000","last_fullsized":"2023-10-09T04:30:42.587607+0000","mapping_epoch":106,"log_start":"110'2600","ondisk_log_start":"110'2600","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":10034,"log_dups_size":0,"ondisk_log_size":10034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:19.007979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107872256,"num_objects":1646,"num_object_clones":0,"num_object_copies":4938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12634,"num_write_kb":456960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":108,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805860+0000","last_change":"2023-10-09T04:29:12.722710+0000","last_active":"2023-10-09T04:30:04.805860+0000","last_peered":"2023-10-09T04:30:04.805860+0000","last_clean":"2023-10-09T04:30:04.805860+0000","last_became_active":"2023-10-09T04:29:12.635166+0000","last_became_peered":"2023-10-09T04:29:12.635166+0000","last_unstale":"2023-10-09T04:30:04.805860+0000","last_undegraded":"2023-10-09T04:30:04.805860+0000","last_fullsized":"2023-10-09T04:30:04.805860+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796538+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:30:04.796538+0000","last_peered":"2023-10-09T04:30:04.796538+0000","last_clean":"2023-10-09T04:30:04.796538+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:30:04.796538+0000","last_undegraded":"2023-10-09T04:30:04.796538+0000","last_fullsized":"2023-10-09T04:30:04.796538+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"127'12729","reported_seq":12760,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:42.530933+0000","last_change":"2023-10-09T04:28:27.604547+0000","last_active":"2023-10-09T04:30:42.530933+0000","last_peered":"2023-10-09T04:30:42.530933+0000","last_clean":"2023-10-09T04:28:26.590593+0000","last_became_active":"2023-10-09T04:28:27.604547+0000","last_became_peered":"2023-10-09T04:28:27.604547+0000","last_unstale":"2023-10-09T04:30:42.530933+0000","last_undegraded":"2023-10-09T04:28:27.593309+0000","last_fullsized":"2023-10-09T04:28:27.593130+0000","mapping_epoch":114,"log_start":"111'2700","ondisk_log_start":"111'2700","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":10029,"log_dups_size":0,"ondisk_log_size":10029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:32.223773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1639,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12729,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1639}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":85,"reported_epoch":126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:04.809054+0000","last_change":"2023-10-09T04:28:27.610412+0000","last_active":"2023-10-09T04:30:04.809054+0000","last_peered":"2023-10-09T04:30:04.809054+0000","last_clean":"2023-10-09T04:28:19.573266+0000","last_became_active":"2023-10-09T04:28:27.610412+0000","last_became_peered":"2023-10-09T04:28:27.610412+0000","last_unstale":"2023-10-09T04:30:04.809054+0000","last_undegraded":"2023-10-09T04:28:27.605925+0000","last_fullsized":"2023-10-09T04:28:27.605804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.795681+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:30:04.795681+0000","last_peered":"2023-10-09T04:30:04.795681+0000","last_clean":"2023-10-09T04:30:04.795681+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:30:04.795681+0000","last_undegraded":"2023-10-09T04:30:04.795681+0000","last_fullsized":"2023-10-09T04:30:04.795681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"127'12565","reported_seq":12596,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:42.563324+0000","last_change":"2023-10-09T04:26:55.435227+0000","last_active":"2023-10-09T04:30:42.563324+0000","last_peered":"2023-10-09T04:30:42.563324+0000","last_clean":"2023-10-09T04:30:42.563324+0000","last_became_active":"2023-10-09T04:26:55.434919+0000","last_became_peered":"2023-10-09T04:26:55.434919+0000","last_unstale":"2023-10-09T04:30:42.563324+0000","last_undegraded":"2023-10-09T04:30:42.563324+0000","last_fullsized":"2023-10-09T04:30:42.563324+0000","mapping_epoch":106,"log_start":"110'2500","ondisk_log_start":"110'2500","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":10065,"log_dups_size":0,"ondisk_log_size":10065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:38.324241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105578496,"num_objects":1611,"num_object_clones":0,"num_object_copies":4833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.796002+0000","last_change":"2023-10-09T04:28:51.527885+0000","last_active":"2023-10-09T04:30:04.796002+0000","last_peered":"2023-10-09T04:30:04.796002+0000","last_clean":"2023-10-09T04:23:18.758933+0000","last_became_active":"2023-10-09T04:28:51.527885+0000","last_became_peered":"2023-10-09T04:28:51.527885+0000","last_unstale":"2023-10-09T04:30:04.796002+0000","last_undegraded":"2023-10-09T04:30:04.796002+0000","last_fullsized":"2023-10-09T04:28:51.525148+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804461+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:30:04.804461+0000","last_peered":"2023-10-09T04:30:04.804461+0000","last_clean":"2023-10-09T04:30:04.804461+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:30:04.804461+0000","last_undegraded":"2023-10-09T04:30:04.804461+0000","last_fullsized":"2023-10-09T04:30:04.804461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"127'12619","reported_seq":12656,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:42.574560+0000","last_change":"2023-10-09T04:28:27.610140+0000","last_active":"2023-10-09T04:30:42.574560+0000","last_peered":"2023-10-09T04:30:42.574560+0000","last_clean":"2023-10-09T04:28:26.568395+0000","last_became_active":"2023-10-09T04:28:27.610140+0000","last_became_peered":"2023-10-09T04:28:27.610140+0000","last_unstale":"2023-10-09T04:30:42.574560+0000","last_undegraded":"2023-10-09T04:28:27.592853+0000","last_fullsized":"2023-10-09T04:28:27.592674+0000","mapping_epoch":114,"log_start":"110'2600","ondisk_log_start":"110'2600","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":10019,"log_dups_size":0,"ondisk_log_size":10019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:18.245131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112853016,"num_objects":1723,"num_object_clones":0,"num_object_copies":5169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1723,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12619,"num_write_kb":458881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1723}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":41,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804525+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:30:04.804525+0000","last_peered":"2023-10-09T04:30:04.804525+0000","last_clean":"2023-10-09T04:30:04.804525+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:30:04.804525+0000","last_undegraded":"2023-10-09T04:30:04.804525+0000","last_fullsized":"2023-10-09T04:30:04.804525+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.797364+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:30:04.797364+0000","last_peered":"2023-10-09T04:30:04.797364+0000","last_clean":"2023-10-09T04:30:04.797364+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:30:04.797364+0000","last_undegraded":"2023-10-09T04:30:04.797364+0000","last_fullsized":"2023-10-09T04:30:04.797364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"127'12516","reported_seq":12547,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:39.169185+0000","last_change":"2023-10-09T04:26:55.433498+0000","last_active":"2023-10-09T04:30:39.169185+0000","last_peered":"2023-10-09T04:30:39.169185+0000","last_clean":"2023-10-09T04:30:39.169185+0000","last_became_active":"2023-10-09T04:26:55.433162+0000","last_became_peered":"2023-10-09T04:26:55.433162+0000","last_unstale":"2023-10-09T04:30:39.169185+0000","last_undegraded":"2023-10-09T04:30:39.169185+0000","last_fullsized":"2023-10-09T04:30:39.169185+0000","mapping_epoch":106,"log_start":"111'4400","ondisk_log_start":"111'4400","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:56.313689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571584,"num_objects":1794,"num_object_clones":0,"num_object_copies":5382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12516,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.795963+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:30:04.795963+0000","last_peered":"2023-10-09T04:30:04.795963+0000","last_clean":"2023-10-09T04:30:04.795963+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:30:04.795963+0000","last_undegraded":"2023-10-09T04:30:04.795963+0000","last_fullsized":"2023-10-09T04:30:04.795963+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":88,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.795582+0000","last_change":"2023-10-09T04:29:12.637984+0000","last_active":"2023-10-09T04:30:04.795582+0000","last_peered":"2023-10-09T04:30:04.795582+0000","last_clean":"2023-10-09T04:30:04.795582+0000","last_became_active":"2023-10-09T04:29:12.637551+0000","last_became_peered":"2023-10-09T04:29:12.637551+0000","last_unstale":"2023-10-09T04:30:04.795582+0000","last_undegraded":"2023-10-09T04:30:04.795582+0000","last_fullsized":"2023-10-09T04:30:04.795582+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:19.948362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.f","version":"127'12407","reported_seq":12439,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:40.304241+0000","last_change":"2023-10-09T04:28:27.611157+0000","last_active":"2023-10-09T04:30:40.304241+0000","last_peered":"2023-10-09T04:30:40.304241+0000","last_clean":"2023-10-09T04:28:26.566546+0000","last_became_active":"2023-10-09T04:28:27.611157+0000","last_became_peered":"2023-10-09T04:28:27.611157+0000","last_unstale":"2023-10-09T04:30:40.304241+0000","last_undegraded":"2023-10-09T04:28:27.598747+0000","last_fullsized":"2023-10-09T04:28:27.598504+0000","mapping_epoch":114,"log_start":"111'4000","ondisk_log_start":"111'4000","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8407,"log_dups_size":0,"ondisk_log_size":8407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1707,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12407,"num_write_kb":451648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1707}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796019+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:30:04.796019+0000","last_peered":"2023-10-09T04:30:04.796019+0000","last_clean":"2023-10-09T04:30:04.796019+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:30:04.796019+0000","last_undegraded":"2023-10-09T04:30:04.796019+0000","last_fullsized":"2023-10-09T04:30:04.796019+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796470+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:30:04.796470+0000","last_peered":"2023-10-09T04:30:04.796470+0000","last_clean":"2023-10-09T04:30:04.796470+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:30:04.796470+0000","last_undegraded":"2023-10-09T04:30:04.796470+0000","last_fullsized":"2023-10-09T04:30:04.796470+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"127'12741","reported_seq":12904,"reported_epoch":127,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T04:30:42.436625+0000","last_change":"2023-10-09T04:29:14.007034+0000","last_active":"2023-10-09T04:30:42.436625+0000","last_peered":"2023-10-09T04:30:42.436625+0000","last_clean":"2023-10-09T04:28:26.567473+0000","last_became_active":"2023-10-09T04:29:13.728615+0000","last_became_peered":"2023-10-09T04:29:13.728615+0000","last_unstale":"2023-10-09T04:30:42.436625+0000","last_undegraded":"2023-10-09T04:29:13.649108+0000","last_fullsized":"2023-10-09T04:29:13.627834+0000","mapping_epoch":121,"log_start":"111'4100","ondisk_log_start":"111'4100","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8641,"log_dups_size":0,"ondisk_log_size":8641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:46.887557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108068864,"num_objects":1649,"num_object_clones":0,"num_object_copies":4947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1631,"num_objects_misplaced":1649,"num_objects_unfound":0,"num_objects_dirty":1649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12741,"num_write_kb":460480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":3932160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1649}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790118+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:30:04.790118+0000","last_peered":"2023-10-09T04:30:04.790118+0000","last_clean":"2023-10-09T04:30:04.790118+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:30:04.790118+0000","last_undegraded":"2023-10-09T04:30:04.790118+0000","last_fullsized":"2023-10-09T04:30:04.790118+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.803042+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:30:04.803042+0000","last_peered":"2023-10-09T04:30:04.803042+0000","last_clean":"2023-10-09T04:30:04.803042+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:30:04.803042+0000","last_undegraded":"2023-10-09T04:30:04.803042+0000","last_fullsized":"2023-10-09T04:30:04.803042+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":498,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.792675+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:30:04.792675+0000","last_peered":"2023-10-09T04:30:04.792675+0000","last_clean":"2023-10-09T04:30:04.792675+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:30:04.792675+0000","last_undegraded":"2023-10-09T04:30:04.792675+0000","last_fullsized":"2023-10-09T04:30:04.792675+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.795547+0000","last_change":"2023-10-09T04:28:51.527252+0000","last_active":"2023-10-09T04:30:04.795547+0000","last_peered":"2023-10-09T04:30:04.795547+0000","last_clean":"2023-10-09T04:23:18.768463+0000","last_became_active":"2023-10-09T04:28:51.527252+0000","last_became_peered":"2023-10-09T04:28:51.527252+0000","last_unstale":"2023-10-09T04:30:04.795547+0000","last_undegraded":"2023-10-09T04:30:04.795547+0000","last_fullsized":"2023-10-09T04:28:51.524832+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:29.101177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.790429+0000","last_change":"2023-10-09T04:28:27.600497+0000","last_active":"2023-10-09T04:30:04.790429+0000","last_peered":"2023-10-09T04:30:04.790429+0000","last_clean":"2023-10-09T04:28:19.561863+0000","last_became_active":"2023-10-09T04:28:27.600497+0000","last_became_peered":"2023-10-09T04:28:27.600497+0000","last_unstale":"2023-10-09T04:30:04.790429+0000","last_undegraded":"2023-10-09T04:30:04.790429+0000","last_fullsized":"2023-10-09T04:28:27.591503+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:26.054422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"127'12502","reported_seq":12543,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:42.459722+0000","last_change":"2023-10-09T04:28:27.644798+0000","last_active":"2023-10-09T04:30:42.459722+0000","last_peered":"2023-10-09T04:30:42.459722+0000","last_clean":"2023-10-09T04:28:26.605951+0000","last_became_active":"2023-10-09T04:28:27.624767+0000","last_became_peered":"2023-10-09T04:28:27.624767+0000","last_unstale":"2023-10-09T04:30:42.459722+0000","last_undegraded":"2023-10-09T04:28:27.603721+0000","last_fullsized":"2023-10-09T04:28:27.597855+0000","mapping_epoch":114,"log_start":"111'3900","ondisk_log_start":"111'3900","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8602,"log_dups_size":0,"ondisk_log_size":8602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:11.563549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107741184,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12502,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":1644}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"127'12469","reported_seq":15587,"reported_epoch":127,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:30:42.467784+0000","last_change":"2023-10-09T04:29:14.031224+0000","last_active":"2023-10-09T04:30:42.467784+0000","last_peered":"2023-10-09T04:30:42.467784+0000","last_clean":"2023-10-09T04:29:11.554878+0000","last_became_active":"2023-10-09T04:29:13.976505+0000","last_became_peered":"2023-10-09T04:29:13.976505+0000","last_unstale":"2023-10-09T04:30:42.467784+0000","last_undegraded":"2023-10-09T04:29:13.673293+0000","last_fullsized":"2023-10-09T04:29:13.625951+0000","mapping_epoch":121,"log_start":"111'3800","ondisk_log_start":"111'3800","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8669,"log_dups_size":0,"ondisk_log_size":8669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:39.303553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104398848,"num_objects":1593,"num_object_clones":0,"num_object_copies":4779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":1593,"num_objects_unfound":0,"num_objects_dirty":1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12469,"num_write_kb":449984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1","objects":-2514},{"shards":"0,1,4","objects":1593}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":587,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:24.508143+0000","last_change":"2023-10-09T04:28:27.605360+0000","last_active":"2023-10-09T04:30:24.508143+0000","last_peered":"2023-10-09T04:30:24.508143+0000","last_clean":"2023-10-09T04:13:52.717465+0000","last_became_active":"2023-10-09T04:28:27.605360+0000","last_became_peered":"2023-10-09T04:28:27.605360+0000","last_unstale":"2023-10-09T04:30:24.508143+0000","last_undegraded":"2023-10-09T04:28:27.591227+0000","last_fullsized":"2023-10-09T04:28:27.591053+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:40.508309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796242+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:30:04.796242+0000","last_peered":"2023-10-09T04:30:04.796242+0000","last_clean":"2023-10-09T04:30:04.796242+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:30:04.796242+0000","last_undegraded":"2023-10-09T04:30:04.796242+0000","last_fullsized":"2023-10-09T04:30:04.796242+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.794668+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:30:04.794668+0000","last_peered":"2023-10-09T04:30:04.794668+0000","last_clean":"2023-10-09T04:30:04.794668+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:30:04.794668+0000","last_undegraded":"2023-10-09T04:30:04.794668+0000","last_fullsized":"2023-10-09T04:30:04.794668+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"127'12666","reported_seq":12701,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:42.564811+0000","last_change":"2023-10-09T04:28:27.609226+0000","last_active":"2023-10-09T04:30:42.564811+0000","last_peered":"2023-10-09T04:30:42.564811+0000","last_clean":"2023-10-09T04:28:26.551404+0000","last_became_active":"2023-10-09T04:28:27.609226+0000","last_became_peered":"2023-10-09T04:28:27.609226+0000","last_unstale":"2023-10-09T04:30:42.564811+0000","last_undegraded":"2023-10-09T04:28:27.592244+0000","last_fullsized":"2023-10-09T04:28:27.592122+0000","mapping_epoch":114,"log_start":"110'2600","ondisk_log_start":"110'2600","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:20.367605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284736,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12666,"num_write_kb":455744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1576}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1234,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.793790+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:30:04.793790+0000","last_peered":"2023-10-09T04:30:04.793790+0000","last_clean":"2023-10-09T04:30:04.793790+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:30:04.793790+0000","last_undegraded":"2023-10-09T04:30:04.793790+0000","last_fullsized":"2023-10-09T04:30:04.793790+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790243+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:30:04.790243+0000","last_peered":"2023-10-09T04:30:04.790243+0000","last_clean":"2023-10-09T04:30:04.790243+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:30:04.790243+0000","last_undegraded":"2023-10-09T04:30:04.790243+0000","last_fullsized":"2023-10-09T04:30:04.790243+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.802907+0000","last_change":"2023-10-09T04:28:27.599081+0000","last_active":"2023-10-09T04:30:04.802907+0000","last_peered":"2023-10-09T04:30:04.802907+0000","last_clean":"2023-10-09T04:28:19.562734+0000","last_became_active":"2023-10-09T04:28:27.599081+0000","last_became_peered":"2023-10-09T04:28:27.599081+0000","last_unstale":"2023-10-09T04:30:04.802907+0000","last_undegraded":"2023-10-09T04:30:04.802907+0000","last_fullsized":"2023-10-09T04:28:27.592506+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:57.226039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":563,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:24.517283+0000","last_change":"2023-10-09T04:28:27.614054+0000","last_active":"2023-10-09T04:30:24.517283+0000","last_peered":"2023-10-09T04:30:24.517283+0000","last_clean":"2023-10-09T04:13:52.699710+0000","last_became_active":"2023-10-09T04:28:27.614054+0000","last_became_peered":"2023-10-09T04:28:27.614054+0000","last_unstale":"2023-10-09T04:30:24.517283+0000","last_undegraded":"2023-10-09T04:28:27.594108+0000","last_fullsized":"2023-10-09T04:28:27.593983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:21.875430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":329,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804918+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:30:04.804918+0000","last_peered":"2023-10-09T04:30:04.804918+0000","last_clean":"2023-10-09T04:30:04.804918+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:30:04.804918+0000","last_undegraded":"2023-10-09T04:30:04.804918+0000","last_fullsized":"2023-10-09T04:30:04.804918+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":95,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.796488+0000","last_change":"2023-10-09T04:28:51.527731+0000","last_active":"2023-10-09T04:30:04.796488+0000","last_peered":"2023-10-09T04:30:04.796488+0000","last_clean":"2023-10-09T04:28:19.566551+0000","last_became_active":"2023-10-09T04:28:51.527731+0000","last_became_peered":"2023-10-09T04:28:51.527731+0000","last_unstale":"2023-10-09T04:30:04.796488+0000","last_undegraded":"2023-10-09T04:30:04.796488+0000","last_fullsized":"2023-10-09T04:28:51.525113+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796943+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:30:04.796943+0000","last_peered":"2023-10-09T04:30:04.796943+0000","last_clean":"2023-10-09T04:30:04.796943+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:30:04.796943+0000","last_undegraded":"2023-10-09T04:30:04.796943+0000","last_fullsized":"2023-10-09T04:30:04.796943+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"127'12494","reported_seq":12528,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:40.280952+0000","last_change":"2023-10-09T04:28:27.609668+0000","last_active":"2023-10-09T04:30:40.280952+0000","last_peered":"2023-10-09T04:30:40.280952+0000","last_clean":"2023-10-09T04:28:26.580793+0000","last_became_active":"2023-10-09T04:28:27.609668+0000","last_became_peered":"2023-10-09T04:28:27.609668+0000","last_unstale":"2023-10-09T04:30:40.280952+0000","last_undegraded":"2023-10-09T04:28:27.605975+0000","last_fullsized":"2023-10-09T04:28:27.605821+0000","mapping_epoch":114,"log_start":"111'3900","ondisk_log_start":"111'3900","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8594,"log_dups_size":0,"ondisk_log_size":8594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112721920,"num_objects":1720,"num_object_clones":0,"num_object_copies":5160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1720,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12494,"num_write_kb":454848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1720}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"127'12515","reported_seq":12550,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:39.070562+0000","last_change":"2023-10-09T04:28:27.600510+0000","last_active":"2023-10-09T04:30:39.070562+0000","last_peered":"2023-10-09T04:30:39.070562+0000","last_clean":"2023-10-09T04:28:26.567988+0000","last_became_active":"2023-10-09T04:28:27.600510+0000","last_became_peered":"2023-10-09T04:28:27.600510+0000","last_unstale":"2023-10-09T04:30:39.070562+0000","last_undegraded":"2023-10-09T04:28:27.591765+0000","last_fullsized":"2023-10-09T04:28:27.591626+0000","mapping_epoch":114,"log_start":"111'4400","ondisk_log_start":"111'4400","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:02.703552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116850688,"num_objects":1783,"num_object_clones":0,"num_object_copies":5349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1783,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12515,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1783}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":671,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805019+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:30:04.805019+0000","last_peered":"2023-10-09T04:30:04.805019+0000","last_clean":"2023-10-09T04:30:04.805019+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:30:04.805019+0000","last_undegraded":"2023-10-09T04:30:04.805019+0000","last_fullsized":"2023-10-09T04:30:04.805019+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":127,"state":"active+clean+remapped","last_fresh":"2023-10-09T04:30:24.503018+0000","last_change":"2023-10-09T04:29:13.767749+0000","last_active":"2023-10-09T04:30:24.503018+0000","last_peered":"2023-10-09T04:30:24.503018+0000","last_clean":"2023-10-09T04:30:24.503018+0000","last_became_active":"2023-10-09T04:29:13.767307+0000","last_became_peered":"2023-10-09T04:29:13.767307+0000","last_unstale":"2023-10-09T04:30:24.503018+0000","last_undegraded":"2023-10-09T04:30:24.503018+0000","last_fullsized":"2023-10-09T04:30:24.503018+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:30.355543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806426+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:30:04.806426+0000","last_peered":"2023-10-09T04:30:04.806426+0000","last_clean":"2023-10-09T04:30:04.806426+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:30:04.806426+0000","last_undegraded":"2023-10-09T04:30:04.806426+0000","last_fullsized":"2023-10-09T04:30:04.806426+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":340,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.808907+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:30:04.808907+0000","last_peered":"2023-10-09T04:30:04.808907+0000","last_clean":"2023-10-09T04:30:04.808907+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:30:04.808907+0000","last_undegraded":"2023-10-09T04:30:04.808907+0000","last_fullsized":"2023-10-09T04:30:04.808907+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.794553+0000","last_change":"2023-10-09T04:28:27.599163+0000","last_active":"2023-10-09T04:30:04.794553+0000","last_peered":"2023-10-09T04:30:04.794553+0000","last_clean":"2023-10-09T04:28:19.563114+0000","last_became_active":"2023-10-09T04:28:27.599163+0000","last_became_peered":"2023-10-09T04:28:27.599163+0000","last_unstale":"2023-10-09T04:30:04.794553+0000","last_undegraded":"2023-10-09T04:30:04.794553+0000","last_fullsized":"2023-10-09T04:28:27.591374+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:37.443801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":70,"reported_epoch":127,"state":"active+undersized","last_fresh":"2023-10-09T04:30:24.508817+0000","last_change":"2023-10-09T04:28:27.610457+0000","last_active":"2023-10-09T04:30:24.508817+0000","last_peered":"2023-10-09T04:30:24.508817+0000","last_clean":"2023-10-09T04:28:19.571509+0000","last_became_active":"2023-10-09T04:28:27.610457+0000","last_became_peered":"2023-10-09T04:28:27.610457+0000","last_unstale":"2023-10-09T04:30:24.508817+0000","last_undegraded":"2023-10-09T04:30:24.508817+0000","last_fullsized":"2023-10-09T04:28:27.598190+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:23.684166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"127'12652","reported_seq":12685,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:40.320193+0000","last_change":"2023-10-09T04:28:27.608939+0000","last_active":"2023-10-09T04:30:40.320193+0000","last_peered":"2023-10-09T04:30:40.320193+0000","last_clean":"2023-10-09T04:28:26.550627+0000","last_became_active":"2023-10-09T04:28:27.608939+0000","last_became_peered":"2023-10-09T04:28:27.608939+0000","last_unstale":"2023-10-09T04:30:40.320193+0000","last_undegraded":"2023-10-09T04:28:27.603725+0000","last_fullsized":"2023-10-09T04:28:27.603461+0000","mapping_epoch":114,"log_start":"111'3800","ondisk_log_start":"111'3800","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117440512,"num_objects":1792,"num_object_clones":0,"num_object_copies":5376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1792,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12652,"num_write_kb":462208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":1792}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":41,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.805864+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:30:04.805864+0000","last_peered":"2023-10-09T04:30:04.805864+0000","last_clean":"2023-10-09T04:30:04.805864+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:30:04.805864+0000","last_undegraded":"2023-10-09T04:30:04.805864+0000","last_fullsized":"2023-10-09T04:30:04.805864+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":349,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790956+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:30:04.790956+0000","last_peered":"2023-10-09T04:30:04.790956+0000","last_clean":"2023-10-09T04:30:04.790956+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:30:04.790956+0000","last_undegraded":"2023-10-09T04:30:04.790956+0000","last_fullsized":"2023-10-09T04:30:04.790956+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804634+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:30:04.804634+0000","last_peered":"2023-10-09T04:30:04.804634+0000","last_clean":"2023-10-09T04:30:04.804634+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:30:04.804634+0000","last_undegraded":"2023-10-09T04:30:04.804634+0000","last_fullsized":"2023-10-09T04:30:04.804634+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"127'12616","reported_seq":12645,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:40.366388+0000","last_change":"2023-10-09T04:26:55.436958+0000","last_active":"2023-10-09T04:30:40.366388+0000","last_peered":"2023-10-09T04:30:40.366388+0000","last_clean":"2023-10-09T04:30:40.366388+0000","last_became_active":"2023-10-09T04:26:55.436284+0000","last_became_peered":"2023-10-09T04:26:55.436284+0000","last_unstale":"2023-10-09T04:30:40.366388+0000","last_undegraded":"2023-10-09T04:30:40.366388+0000","last_fullsized":"2023-10-09T04:30:40.366388+0000","mapping_epoch":106,"log_start":"111'4200","ondisk_log_start":"111'4200","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8416,"log_dups_size":0,"ondisk_log_size":8416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12616,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":174,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.797204+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:30:04.797204+0000","last_peered":"2023-10-09T04:30:04.797204+0000","last_clean":"2023-10-09T04:30:04.797204+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:30:04.797204+0000","last_undegraded":"2023-10-09T04:30:04.797204+0000","last_fullsized":"2023-10-09T04:30:04.797204+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.806332+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:30:04.806332+0000","last_peered":"2023-10-09T04:30:04.806332+0000","last_clean":"2023-10-09T04:30:04.806332+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:30:04.806332+0000","last_undegraded":"2023-10-09T04:30:04.806332+0000","last_fullsized":"2023-10-09T04:30:04.806332+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.796866+0000","last_change":"2023-10-09T04:28:27.605015+0000","last_active":"2023-10-09T04:30:04.796866+0000","last_peered":"2023-10-09T04:30:04.796866+0000","last_clean":"2023-10-09T04:28:19.571550+0000","last_became_active":"2023-10-09T04:28:27.605015+0000","last_became_peered":"2023-10-09T04:28:27.605015+0000","last_unstale":"2023-10-09T04:30:04.796866+0000","last_undegraded":"2023-10-09T04:30:04.796866+0000","last_fullsized":"2023-10-09T04:28:27.598518+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"127'12385","reported_seq":12422,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-09T04:30:39.127728+0000","last_change":"2023-10-09T04:28:51.540229+0000","last_active":"2023-10-09T04:30:39.127728+0000","last_peered":"2023-10-09T04:30:39.127728+0000","last_clean":"2023-10-09T04:30:39.127728+0000","last_became_active":"2023-10-09T04:28:51.530787+0000","last_became_peered":"2023-10-09T04:28:51.530787+0000","last_unstale":"2023-10-09T04:30:39.127728+0000","last_undegraded":"2023-10-09T04:30:39.127728+0000","last_fullsized":"2023-10-09T04:30:39.127728+0000","mapping_epoch":117,"log_start":"111'4200","ondisk_log_start":"111'4200","created":106,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:26:54.410798+0000","last_clean_scrub_stamp":"2023-10-09T04:26:54.410798+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:20.235639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119865344,"num_objects":1829,"num_object_clones":0,"num_object_copies":5487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12385,"num_write_kb":454848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":67,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804550+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:30:04.804550+0000","last_peered":"2023-10-09T04:30:04.804550+0000","last_clean":"2023-10-09T04:30:04.804550+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:30:04.804550+0000","last_undegraded":"2023-10-09T04:30:04.804550+0000","last_fullsized":"2023-10-09T04:30:04.804550+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+undersized","last_fresh":"2023-10-09T04:30:04.794170+0000","last_change":"2023-10-09T04:28:27.600435+0000","last_active":"2023-10-09T04:30:04.794170+0000","last_peered":"2023-10-09T04:30:04.794170+0000","last_clean":"2023-10-09T04:28:19.572818+0000","last_became_active":"2023-10-09T04:28:27.600435+0000","last_became_peered":"2023-10-09T04:28:27.600435+0000","last_unstale":"2023-10-09T04:30:04.794170+0000","last_undegraded":"2023-10-09T04:30:04.794170+0000","last_fullsized":"2023-10-09T04:28:27.592390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:45.015770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":32,"reported_epoch":126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:04.797051+0000","last_change":"2023-10-09T04:28:27.600816+0000","last_active":"2023-10-09T04:30:04.797051+0000","last_peered":"2023-10-09T04:30:04.797051+0000","last_clean":"2023-10-09T04:11:28.922252+0000","last_became_active":"2023-10-09T04:28:27.600816+0000","last_became_peered":"2023-10-09T04:28:27.600816+0000","last_unstale":"2023-10-09T04:30:04.797051+0000","last_undegraded":"2023-10-09T04:28:27.597550+0000","last_fullsized":"2023-10-09T04:28:27.597399+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.790712+0000","last_change":"2023-10-09T04:29:12.632856+0000","last_active":"2023-10-09T04:30:04.790712+0000","last_peered":"2023-10-09T04:30:04.790712+0000","last_clean":"2023-10-09T04:30:04.790712+0000","last_became_active":"2023-10-09T04:29:12.632465+0000","last_became_peered":"2023-10-09T04:29:12.632465+0000","last_unstale":"2023-10-09T04:30:04.790712+0000","last_undegraded":"2023-10-09T04:30:04.790712+0000","last_fullsized":"2023-10-09T04:30:04.790712+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:26.854929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":32,"reported_epoch":126,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:04.797121+0000","last_change":"2023-10-09T04:28:27.606662+0000","last_active":"2023-10-09T04:30:04.797121+0000","last_peered":"2023-10-09T04:30:04.797121+0000","last_clean":"2023-10-09T04:11:28.923128+0000","last_became_active":"2023-10-09T04:28:27.606662+0000","last_became_peered":"2023-10-09T04:28:27.606662+0000","last_unstale":"2023-10-09T04:30:04.797121+0000","last_undegraded":"2023-10-09T04:28:27.599069+0000","last_fullsized":"2023-10-09T04:28:27.598215+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.804294+0000","last_change":"2023-10-09T04:29:12.632656+0000","last_active":"2023-10-09T04:30:04.804294+0000","last_peered":"2023-10-09T04:30:04.804294+0000","last_clean":"2023-10-09T04:30:04.804294+0000","last_became_active":"2023-10-09T04:29:12.632273+0000","last_became_peered":"2023-10-09T04:29:12.632273+0000","last_unstale":"2023-10-09T04:30:04.804294+0000","last_undegraded":"2023-10-09T04:30:04.804294+0000","last_fullsized":"2023-10-09T04:30:04.804294+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:27.569523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":30,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796929+0000","last_change":"2023-10-09T04:28:51.529660+0000","last_active":"2023-10-09T04:30:04.796929+0000","last_peered":"2023-10-09T04:30:04.796929+0000","last_clean":"2023-10-09T04:30:04.796929+0000","last_became_active":"2023-10-09T04:28:51.529256+0000","last_became_peered":"2023-10-09T04:28:51.529256+0000","last_unstale":"2023-10-09T04:30:04.796929+0000","last_undegraded":"2023-10-09T04:30:04.796929+0000","last_fullsized":"2023-10-09T04:30:04.796929+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.793704+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:30:04.793704+0000","last_peered":"2023-10-09T04:30:04.793704+0000","last_clean":"2023-10-09T04:30:04.793704+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:30:04.793704+0000","last_undegraded":"2023-10-09T04:30:04.793704+0000","last_fullsized":"2023-10-09T04:30:04.793704+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":89,"reported_epoch":127,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:30:24.506696+0000","last_change":"2023-10-09T04:28:27.606014+0000","last_active":"2023-10-09T04:30:24.506696+0000","last_peered":"2023-10-09T04:30:24.506696+0000","last_clean":"2023-10-09T04:28:19.571450+0000","last_became_active":"2023-10-09T04:28:27.606014+0000","last_became_peered":"2023-10-09T04:28:27.606014+0000","last_unstale":"2023-10-09T04:30:24.506696+0000","last_undegraded":"2023-10-09T04:28:27.597046+0000","last_fullsized":"2023-10-09T04:28:27.594857+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:07.466276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-09T04:30:04.796994+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:30:04.796994+0000","last_peered":"2023-10-09T04:30:04.796994+0000","last_clean":"2023-10-09T04:30:04.796994+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:30:04.796994+0000","last_undegraded":"2023-10-09T04:30:04.796994+0000","last_fullsized":"2023-10-09T04:30:04.796994+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1789591576,"num_objects":27308,"num_object_clones":0,"num_object_copies":81924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17729,"num_objects_misplaced":3242,"num_objects_unfound":0,"num_objects_dirty":27308,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201150,"num_write_kb":7310593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1600,"num_bytes_recovered":104857600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8194760704,"data_stored":8194752560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143450,"ondisk_log_size":143450,"up":39,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13247,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430475,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":891564,"kb_used_data":586148,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92853588,"statfs":{"total":95995035648,"available":95082074112,"internally_reserved":0,"allocated":600215552,"data_stored":599694227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:39 2023","interfaces":[{"interface":"back","average":{"1min":13.199,"5min":3.419,"15min":1.736},"min":{"1min":0.29599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":250.65700000000001,"5min":250.65700000000001,"15min":250.65700000000001},"last":0.375},{"interface":"front","average":{"1min":13.388,"5min":3.7040000000000002,"15min":1.8460000000000001},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":251.214,"5min":251.214,"15min":251.214},"last":0.63200000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.105,"15min":0.98799999999999999},"min":{"1min":0.30399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.8369999999999997,"5min":7.282,"15min":7.282},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.165,"5min":1.3680000000000001,"15min":1.081},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.71,"5min":5.71,"15min":5.71},"last":1.5109999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.0900000000000001,"15min":0.97999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.2480000000000002,"5min":4.8310000000000004,"15min":4.8310000000000004},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.577,"5min":1.2130000000000001,"15min":1.0649999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.383,"5min":7.383,"15min":7.383},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.079,"15min":1.044},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.318},"max":{"1min":4.516,"5min":5.1820000000000004,"15min":5.1820000000000004},"last":0.435},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.232,"15min":1.1020000000000001},"min":{"1min":0.40200000000000002,"5min":0.37,"15min":0.37},"max":{"1min":4.524,"5min":7.3399999999999999,"15min":7.3399999999999999},"last":0.443}]},{"osd":4,"last update":"Mon Oct 9 04:30:17 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":1.379,"15min":1.0680000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.1529999999999996,"5min":5.7969999999999997,"15min":5.7969999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.0920000000000001,"15min":1.01},"min":{"1min":0.374,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.823,"5min":5.2430000000000003,"15min":5.2430000000000003},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.0600000000000001,"15min":1.0089999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.0209999999999999,"5min":4.0209999999999999,"15min":4.0209999999999999},"last":5.4470000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.2,"15min":1.0800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":1.9219999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:30:17 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.3140000000000001,"15min":1.143},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.2320000000000002,"5min":5.9820000000000002,"15min":5.9820000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.042,"15min":1.044},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.2930000000000001,"5min":3.1219999999999999,"15min":3.1219999999999999},"last":0.61499999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626708,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":760800,"kb_used_data":459288,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92984352,"statfs":{"total":95995035648,"available":95215976448,"internally_reserved":0,"allocated":470310912,"data_stored":469801724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4860000000000002,"5min":3.5939999999999999,"15min":1.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":51.878999999999998,"5min":147.25800000000001,"15min":147.25800000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":4.1900000000000004,"5min":3.5259999999999998,"15min":1.7450000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":52.262999999999998,"5min":147.37,"15min":147.37},"last":0.46899999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.0860000000000001,"15min":0.95899999999999996},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.371,"5min":8.2690000000000001,"15min":8.2690000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.107,"15min":0.97299999999999998},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.0090000000000003,"5min":10.099,"15min":10.099},"last":0.42799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.605,"5min":4.0170000000000003,"15min":1.98},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":51.813000000000002,"5min":141.93899999999999,"15min":141.93899999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":4.4349999999999996,"5min":3.8399999999999999,"15min":1.895},"min":{"1min":0.35399999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":51.942,"5min":141.898,"15min":141.898},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.1339999999999999,"15min":0.98899999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.3460000000000001,"5min":10.029,"15min":10.029},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.0580000000000001,"15min":1.046},"min":{"1min":0.39600000000000002,"5min":0.378,"15min":0.378},"max":{"1min":3.3570000000000002,"5min":6.0519999999999996,"15min":6.0519999999999996},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.1439999999999999,"15min":0.98799999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.5899999999999999,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.0669999999999999,"15min":1.0009999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.7000000000000002,"5min":11.018000000000001,"15min":11.018000000000001},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.0509999999999999,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.7120000000000002,"5min":3.7120000000000002,"15min":3.9180000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.0670000000000002,"5min":1.242,"15min":1.0780000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1719999999999999,"15min":1.1120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.6519999999999992,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.2789999999999999,"15min":1.2070000000000001},"min":{"1min":0.39100000000000001,"5min":0.375,"15min":0.375},"max":{"1min":5.3860000000000001,"5min":8.9139999999999997,"15min":8.9139999999999997},"last":0.57699999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958624,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":690636,"kb_used_data":586756,"kb_used_omap":6,"kb_used_meta":103865,"kb_avail":93054516,"statfs":{"total":95995035648,"available":95287824384,"internally_reserved":0,"allocated":600838144,"data_stored":600231037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":106357772},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:02 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":3.3580000000000001,"15min":1.5700000000000001},"min":{"1min":0.34599999999999997,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.3130000000000006,"5min":158.435,"15min":158.435},"last":4.9779999999999998},{"interface":"front","average":{"1min":3.577,"5min":3.4780000000000002,"15min":1.657},"min":{"1min":0.31900000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.5920000000000005,"5min":159.38999999999999,"15min":159.38999999999999},"last":4.9349999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:30:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":3.4209999999999998,"15min":1.587},"min":{"1min":0.44400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.0919999999999996,"5min":157.70400000000001,"15min":157.70400000000001},"last":4.181},{"interface":"front","average":{"1min":1.569,"5min":3.3410000000000002,"15min":1.631},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.1139999999999999,"5min":157.64099999999999,"15min":157.64099999999999},"last":4.2789999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":1.2370000000000001,"15min":0.96299999999999997},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.851,"5min":7.851,"15min":7.851},"last":3.8380000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.0609999999999999,"15min":0.81000000000000005},"min":{"1min":0.27500000000000002,"5min":0.249,"15min":0.249},"max":{"1min":7.556,"5min":7.556,"15min":7.556},"last":4.2130000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:30:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":1.3939999999999999,"15min":1.079},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.372999999999999,"5min":10.372999999999999,"15min":10.372999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.2069999999999999,"5min":1.4159999999999999,"15min":1.1699999999999999},"min":{"1min":0.40999999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":9.2850000000000001,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":4.1079999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.9449999999999998,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.99099999999999999,"15min":0.98899999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7069999999999999,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:30:02 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.1819999999999999,"15min":1.0820000000000001},"min":{"1min":0.38600000000000001,"5min":0.374,"15min":0.374},"max":{"1min":7.6029999999999998,"5min":7.6029999999999998,"15min":7.6029999999999998},"last":2.1179999999999999},{"interface":"front","average":{"1min":2.121,"5min":1.3069999999999999,"15min":1.119},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3710000000000004,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:30:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.3660000000000001,"15min":1.1279999999999999},"min":{"1min":0.40699999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":6.3399999999999999,"5min":6.3399999999999999,"15min":6.3399999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.4450000000000001,"15min":1.2290000000000001},"min":{"1min":0.46200000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":9.2390000000000008,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":4.1459999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122150,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000276,"kb_used_data":860044,"kb_used_omap":9,"kb_used_meta":140022,"kb_avail":92744876,"statfs":{"total":95995035648,"available":94970753024,"internally_reserved":0,"allocated":880685056,"data_stored":880071643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":143383025},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:30:08 2023","interfaces":[{"interface":"back","average":{"1min":4.3479999999999999,"5min":1.369,"15min":0.88600000000000001},"min":{"1min":0.22700000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":43.216000000000001,"5min":43.216000000000001,"15min":43.216000000000001},"last":3.504},{"interface":"front","average":{"1min":4.0650000000000004,"5min":1.4850000000000001,"15min":0.94099999999999995},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":43.091999999999999,"5min":43.091999999999999,"15min":43.091999999999999},"last":0.60699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.7649999999999999,"15min":1.0249999999999999},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.843,"5min":37.453000000000003,"15min":37.453000000000003},"last":4.5730000000000004},{"interface":"front","average":{"1min":1.881,"5min":1.502,"15min":1.014},"min":{"1min":0.27000000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.9400000000000004,"5min":25.178999999999998,"15min":25.178999999999998},"last":0.86599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":0.89500000000000002,"15min":0.84899999999999998},"min":{"1min":0.32900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.5419999999999998,"5min":7.5419999999999998,"15min":7.5419999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.042,"5min":1.012,"15min":0.81000000000000005},"min":{"1min":0.29299999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.4269999999999996,"5min":7.5250000000000004,"15min":7.5250000000000004},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.663,"15min":1.2609999999999999},"min":{"1min":0.34499999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.1649999999999991,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.669,"5min":1.246,"15min":1.103},"min":{"1min":0.372,"5min":0.308,"15min":0.308},"max":{"1min":6.1719999999999997,"5min":7.5880000000000001,"15min":7.5880000000000001},"last":0.83099999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.034,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":3.6339999999999999,"5min":3.6339999999999999,"15min":3.6339999999999999},"last":0.441},{"interface":"front","average":{"1min":1.238,"5min":1.0369999999999999,"15min":1.038},"min":{"1min":0.65500000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":3.343,"5min":3.343,"15min":3.343},"last":2.0150000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.266,"15min":1.103},"min":{"1min":0.38200000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.2080000000000002,"5min":7.508,"15min":7.508},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.419,"15min":1.1479999999999999},"min":{"1min":0.40799999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.1449999999999996,"5min":23.994,"15min":23.994},"last":1.677}]},{"osd":7,"last update":"Mon Oct 9 04:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.7150000000000001,"15min":1.3149999999999999},"min":{"1min":0.41299999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.4990000000000006,"5min":10.528,"15min":10.528},"last":0.443},{"interface":"front","average":{"1min":2.2040000000000002,"5min":1.8959999999999999,"15min":1.3839999999999999},"min":{"1min":0.38200000000000001,"5min":0.371,"15min":0.371},"max":{"1min":8.2149999999999999,"5min":9.7029999999999994,"15min":9.7029999999999994},"last":0.38700000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795099,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":763772,"kb_used_data":637684,"kb_used_omap":11,"kb_used_meta":126004,"kb_avail":92981380,"statfs":{"total":95995035648,"available":95212933120,"internally_reserved":0,"allocated":652988416,"data_stored":652394068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":129028138},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.2010000000000001,"15min":0.85699999999999998},"min":{"1min":0.22,"5min":0.191,"15min":0.191},"max":{"1min":6.4729999999999999,"5min":7.577,"15min":7.577},"last":0.217},{"interface":"front","average":{"1min":1.792,"5min":1.3740000000000001,"15min":0.88800000000000001},"min":{"1min":0.23499999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.3529999999999998,"5min":8.3529999999999998,"15min":8.3529999999999998},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.27,"15min":0.91200000000000003},"min":{"1min":0.34999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.1769999999999996,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":0.25},{"interface":"front","average":{"1min":2.222,"5min":1.5609999999999999,"15min":0.99399999999999999},"min":{"1min":0.318,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.550000000000001,"5min":11.550000000000001,"15min":11.550000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.159,"15min":0.93700000000000006},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5330000000000004,"5min":10.044,"15min":10.044},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.254,"15min":0.98199999999999998},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":6.7709999999999999,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":6.093}]},{"osd":4,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.5649999999999999,"15min":1.1519999999999999},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.9210000000000003,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":6.0659999999999998},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.865,"15min":1.258},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.984,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":4.3799999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0409999999999999,"15min":1.0309999999999999},"min":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.113,"5min":0.98499999999999999,"15min":0.96799999999999997},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":4.649}]},{"osd":6,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.829,"15min":1.2869999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.9500000000000002,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":4.4180000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.2210000000000001,"15min":1.1140000000000001},"min":{"1min":0.41299999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.7039999999999997,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.377}]},{"osd":7,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":1.502,"15min":1.256},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.665,"5min":8.2430000000000003,"15min":8.2430000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.5800000000000001,"15min":1.2509999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.952,"5min":8.952,"15min":8.952},"last":6.0259999999999998}]}]},{"osd":3,"up_from":103,"seq":442381631570,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":423596,"kb_used_data":359972,"kb_used_omap":5,"kb_used_meta":63610,"kb_avail":93321556,"statfs":{"total":95995035648,"available":95561273344,"internally_reserved":0,"allocated":368611328,"data_stored":368031953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":65136814},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.1880000000000002,"15min":1.2509999999999999},"min":{"1min":0.38900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.284000000000001,"5min":61.966999999999999,"15min":61.966999999999999},"last":4.3659999999999997},{"interface":"front","average":{"1min":2.335,"5min":2.0600000000000001,"15min":1.1479999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.0020000000000007,"5min":62.052999999999997,"15min":62.052999999999997},"last":4.4630000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.6539999999999999,"15min":1.034},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.6849999999999996,"5min":61.395000000000003,"15min":61.395000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.663,"5min":1.625,"15min":1.1040000000000001},"min":{"1min":0.23599999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.9740000000000002,"5min":61.179000000000002,"15min":61.179000000000002},"last":0.72399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.8999999999999999,"15min":1.1799999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":9.4610000000000003,"5min":63.274999999999999,"15min":63.274999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.085,"5min":1.784,"15min":1.2470000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1200000000000001,"5min":63.107999999999997,"15min":63.107999999999997},"last":0.76400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.0309999999999999,"15min":1.077},"min":{"1min":0.36599999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0190000000000001,"5min":6.0190000000000001,"15min":6.0190000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.0820000000000001,"15min":1.0289999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.4620000000000002,"5min":4.1820000000000004,"15min":4.1820000000000004},"last":0.438}]},{"osd":5,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.96499999999999997,"15min":1.0229999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":1.917},{"interface":"front","average":{"1min":0.872,"5min":0.997,"15min":1.056},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.996,"5min":1.996,"15min":1.996},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.91400000000000003,"15min":1.01},"min":{"1min":0.33700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.335,"5min":3.528,"15min":3.528},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.252,"5min":1.0580000000000001,"15min":1.0920000000000001},"min":{"1min":0.40799999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.0039999999999996,"5min":6.0039999999999996,"15min":6.0039999999999996},"last":0.57099999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.177,"15min":1.155},"min":{"1min":0.434,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":6.0359999999999996,"5min":6.0359999999999996,"15min":6.0359999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.159,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.8209999999999997,"5min":6.8209999999999997,"15min":6.8209999999999997},"last":0.78900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019172,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1463968,"kb_used_data":1154456,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92281184,"statfs":{"total":95995035648,"available":94495932416,"internally_reserved":0,"allocated":1182162944,"data_stored":1181653269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.8049999999999999,"15min":1.1719999999999999},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.1230000000000002,"5min":48.450000000000003,"15min":48.450000000000003},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.7430000000000001,"5min":1.9910000000000001,"15min":1.2669999999999999},"min":{"1min":0.315,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.052,"5min":49.283000000000001,"15min":49.283000000000001},"last":0.371}]},{"osd":1,"last update":"Mon Oct 9 04:30:01 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.3280000000000001,"15min":1.048},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.016,"5min":6.1299999999999999,"15min":6.1299999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.5139999999999998,"5min":1.53,"15min":1.107},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.0809999999999995,"5min":9.9399999999999995,"15min":9.9399999999999995},"last":0.92200000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":1.998,"15min":1.2829999999999999},"min":{"1min":0.34799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.577,"5min":48.323,"15min":48.323},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.298,"15min":1.4059999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":11.064,"5min":48.289000000000001,"15min":48.289000000000001},"last":4.056}]},{"osd":3,"last update":"Mon Oct 9 04:30:07 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.1080000000000001,"15min":1.0600000000000001},"min":{"1min":0.41199999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.1500000000000004,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.2120000000000002,"5min":1.2170000000000001,"15min":1.1299999999999999},"min":{"1min":0.42099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.5440000000000005,"5min":9.5440000000000005,"15min":9.5440000000000005},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98999999999999999,"15min":0.95399999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.087,"15min":0.95399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":1.4770000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.96699999999999997,"15min":0.96599999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.5059999999999998,"5min":4.8540000000000001,"15min":4.8540000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.403,"5min":1.28,"15min":1.113},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":15.977,"5min":15.977,"15min":15.977},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 04:30:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":1.389,"15min":1.2110000000000001},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.5709999999999997,"5min":11.725,"15min":11.725},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.1830000000000001,"15min":1.1339999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.3220000000000001,"5min":6.5730000000000004,"15min":6.5730000000000004},"last":0.623}]}]},{"osd":5,"up_from":36,"seq":154618822912,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3745056,"kb_used_data":3439640,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90000096,"statfs":{"total":95995035648,"available":92160098304,"internally_reserved":0,"allocated":3522191360,"data_stored":3521698716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8919999999999999,"5min":1.474,"15min":1.073},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":53.731000000000002,"5min":53.731000000000002,"15min":53.731000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":1.5429999999999999,"15min":1.0620000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":53.222999999999999,"5min":53.222999999999999,"15min":53.222999999999999},"last":2.359}]},{"osd":1,"last update":"Mon Oct 9 04:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.98099999999999998,"15min":0.997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.252},{"interface":"front","average":{"1min":1.038,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.94699999999999995,"15min":0.94099999999999995},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":2.4430000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":2.5390000000000001,"5min":2.5390000000000001,"15min":2.5390000000000001},"last":2.4649999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.008,"15min":0.99399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":2.2789999999999999,"5min":2.2789999999999999,"15min":2.2789999999999999},"last":0.378},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.1240000000000001,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5140000000000002,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":2.4180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:43 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96299999999999997,"15min":0.94899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9100000000000001,"5min":3.9100000000000001,"15min":3.9100000000000001},"last":5.6479999999999997},{"interface":"front","average":{"1min":1.304,"5min":1.0900000000000001,"15min":1.012},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":3.9990000000000001},"last":2.395}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.998,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.9849999999999999,"5min":3.9849999999999999,"15min":3.9849999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0740000000000001,"15min":1.0660000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.847,"5min":1.847,"15min":2.5840000000000001},"last":0.32800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1379999999999999,"15min":1.113},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.431,"5min":3.992,"15min":3.992},"last":0.312},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.1120000000000001,"15min":1.095},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":1.4610000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":860815360,"data_stored":860815360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":586153984,"data_stored":586153984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":643698688,"data_stored":643694616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":357629952,"data_stored":357629952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1174077440,"data_stored":1174077440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3527147520,"data_stored":3527147520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":463667200,"data_stored":463667200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":581570560,"data_stored":581566488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:30:48.086 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T04:30:48.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:30:48.111 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:30:48.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:48.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:48.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:48.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:48.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:48.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:48.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:48.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:48.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.773 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:48 smithi082 bash[77063]: cluster 2023-10-09T04:30:47.439403+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v446: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:48.773 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:48 smithi082 bash[79733]: cluster 2023-10-09T04:30:47.439403+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v446: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:48.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:48.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:48.915 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:48 smithi153 bash[59565]: cluster 2023-10-09T04:30:47.439403+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v446: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 16871/76656 objects degraded (22.009%); 3242/76656 objects misplaced (4.229%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:48.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:48.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:48.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:49.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:49.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.154 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20168 5.e 5:73f6c3c2:::benchmark_data_smithi082_106363_object20167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 205.28739689700001, 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12474445000000001, 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20168 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.155 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859968+0000", 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8635e-05 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859986+0000", 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4649999999999999e-06 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.157 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859993+0000", 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7000000000000002e-06 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859996+0000", 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3295999999999999e-05 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860050+0000", 2023-10-09T04:30:49.158 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014821499999999999 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860198+0000", 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0001594 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860357+0000", 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024722350000000001 2023-10-09T04:30:49.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.862830+0000", 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0014452270000000001 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.864275+0000", 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.120165037 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984440+0000", 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017054900000000001 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984610+0000", 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000102767 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.161 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984713+0000", 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12474445000000001 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20184 5.e 5:7994a2da:::benchmark_data_smithi082_106363_object20183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 205.27811583499999, 2023-10-09T04:30:49.162 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115792874, 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20184 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.163 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999948 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869244+0000", 2023-10-09T04:30:49.164 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2075000000000004e-05 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869317+0000", 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2318e-05 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869329+0000", 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5759999999999996e-06 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869334+0000", 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2394000000000001e-05 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869377+0000", 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010204600000000001 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869479+0000", 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011170100000000001 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869591+0000", 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000800163 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.870391+0000", 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00032963699999999997 2023-10-09T04:30:49.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.870720+0000", 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114094243 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984815+0000", 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000125501 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984940+0000", 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000102484 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.985043+0000", 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115792874 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.169 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:32660 5.e 5:7353f940:::benchmark_data_smithi082_106363_object32659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:30.984070+0000", 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.16329531299999, 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10087945600000001, 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 32660 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.170 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984070+0000", 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984070+0000", 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.171 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984069+0000", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3522999999999994e-05 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984143+0000", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0434999999999999e-05 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984153+0000", 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.336e-06 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.172 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.984160+0000", 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026846140000000001 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986844+0000", 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000136627 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.173 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986981+0000", 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000168338 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.987149+0000", 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00594116 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993090+0000", 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0605000000000006e-05 2023-10-09T04:30:49.174 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993161+0000", 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091666088000000007 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.084827+0000", 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8905999999999994e-05 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.175 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.084896+0000", 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3798999999999998e-05 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.084950+0000", 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10087945600000001 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.176 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:32665 5.e 5:7837b631:::benchmark_data_smithi082_106363_object32664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:30.986492+0000", 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.16087370700001, 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.098609520000000006, 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 32665 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.177 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986492+0000", 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986492+0000", 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.178 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986491+0000", 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7988999999999998e-05 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986529+0000", 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3530000000000002e-06 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986535+0000", 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6689999999999997e-06 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.179 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.986539+0000", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0019360969999999999 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.988475+0000", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000153084 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.988628+0000", 2023-10-09T04:30:49.180 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020086600000000001 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.988829+0000", 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0047766589999999999 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993606+0000", 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002944145 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.181 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996550+0000", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088450954999999998 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085001+0000", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5118999999999999e-05 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085056+0000", 2023-10-09T04:30:49.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5176000000000003e-05 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085101+0000", 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.098609520000000006 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:32681 5.e 5:70f63408:::benchmark_data_smithi082_106363_object32680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.183 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:30.993816+0000", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.153550026, 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091432322999999996, 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 32681 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993816+0000", 2023-10-09T04:30:49.184 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993816+0000", 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993814+0000", 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7717999999999999e-05 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.185 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993872+0000", 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6929e-05 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993889+0000", 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5100000000000001e-06 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.186 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993896+0000", 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0256000000000002e-05 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.993937+0000", 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011137999999999999 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.994048+0000", 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011738700000000001 2023-10-09T04:30:49.187 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.994165+0000", 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001328367 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.995494+0000", 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040628349999999999 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.188 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.999557+0000", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.085596051000000006 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085153+0000", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1687999999999997e-05 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085204+0000", 2023-10-09T04:30:49.189 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3504000000000002e-05 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085248+0000", 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091432322999999996 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:32685 5.e 5:7ae7f307:::benchmark_data_smithi082_106363_object32684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.190 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:30.996662+0000", 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.15070345999999, 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088730162000000001, 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 32685 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.191 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996662+0000", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996662+0000", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996660+0000", 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5612999999999998e-05 2023-10-09T04:30:49.192 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996716+0000", 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6844999999999999e-05 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996732+0000", 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5420000000000002e-06 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.193 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996739+0000", 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9725000000000002e-05 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996789+0000", 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000150734 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.996939+0000", 2023-10-09T04:30:49.194 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000152521 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.997092+0000", 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026091980000000001 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.999701+0000", 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001220219 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.195 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.000921+0000", 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084376487999999999 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085298+0000", 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0776000000000003e-05 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.196 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085349+0000", 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3652999999999997e-05 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.085392+0000", 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088730162000000001 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.197 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:32693 5.e 5:77778678:::benchmark_data_smithi082_106363_object32692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:31.001329+0000", 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.146036366, 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084881074000000001, 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 32693 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.198 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001329+0000", 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001329+0000", 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001328+0000", 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6323000000000003e-05 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001384+0000", 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2088e-05 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001396+0000", 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4049999999999999e-06 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.200 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001401+0000", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7979999999999998e-05 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001449+0000", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014986700000000001 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001599+0000", 2023-10-09T04:30:49.201 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020819 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.001807+0000", 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001172204 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.002980+0000", 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019361199999999999 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.202 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.003173+0000", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.082939416000000002 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.086113+0000", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1445e-05 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.086164+0000", 2023-10-09T04:30:49.203 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6128000000000003e-05 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:31.086210+0000", 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084881074000000001 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.204 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71653 5.e 5:7f952d4f:::benchmark_data_smithi082_106363_object71652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.205545+0000", 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 167.941820979, 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11021012299999999, 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71653 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.205 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205545+0000", 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205545+0000", 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.206 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205542+0000", 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5834999999999997e-05 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205588+0000", 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1955e-05 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205600+0000", 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3679999999999998e-06 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.207 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205606+0000", 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7626e-05 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205664+0000", 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013713399999999999 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.208 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205801+0000", 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000167656 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205969+0000", 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00778782 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213756+0000", 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020905229999999999 2023-10-09T04:30:49.209 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.215847+0000", 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099645964000000004 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.315493+0000", 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015050400000000001 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.210 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.315643+0000", 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111338 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.315755+0000", 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11021012299999999 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.211 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71654 5.e 5:7baa3dd3:::benchmark_data_smithi082_106363_object71653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 167.94173552199999, 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115496826, 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71654 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.212 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205629+0000", 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5355000000000002e-05 2023-10-09T04:30:49.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205675+0000", 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3310000000000004e-06 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205683+0000", 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3079999999999998e-06 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205688+0000", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029010419999999999 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.208589+0000", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015918799999999999 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.208748+0000", 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027670860000000002 2023-10-09T04:30:49.215 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211515+0000", 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0043369649999999999 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.215852+0000", 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00051018199999999997 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.216 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216363+0000", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.104664519 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321027+0000", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1848999999999998e-05 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321079+0000", 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7880999999999998e-05 2023-10-09T04:30:49.217 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321127+0000", 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115496826 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71657 5.e 5:7b75b376:::benchmark_data_smithi082_106363_object71656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:30:49.218 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 167.94167075199999, 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115567221, 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71657 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:30:49.219 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205694+0000", 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5553000000000003e-05 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205730+0000", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2259999999999999e-06 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205737+0000", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2429999999999996e-06 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205742+0000", 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080320359999999993 2023-10-09T04:30:49.221 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213774+0000", 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103989 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213878+0000", 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010159500000000001 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.222 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213980+0000", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002066713 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216047+0000", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0037985639999999999 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.219845+0000", 2023-10-09T04:30:49.223 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.101326286 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321171+0000", 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1718000000000002e-05 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321223+0000", 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8986999999999998e-05 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321262+0000", 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115567221 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:86151 5.e 5:7978ffe2:::benchmark_data_smithi082_106363_object86150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:15.168777+0000", 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 153.97858824900001, 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.061821480999999998, 2023-10-09T04:30:49.225 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 86151 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168777+0000", 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.226 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168777+0000", 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168777+0000", 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0001063 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.227 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168883+0000", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0879999999999997e-06 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168890+0000", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0899999999999998e-06 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168894+0000", 2023-10-09T04:30:49.228 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.427e-05 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.168949+0000", 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019179700000000001 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.169140+0000", 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00038738899999999997 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.229 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.169528+0000", 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0066281600000000001 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176156+0000", 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040540430000000002 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.230 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.180210+0000", 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.050310105000000001 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.230520+0000", 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8651e-05 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.230559+0000", 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0151000000000002e-05 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.231 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.230599+0000", 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.061821480999999998 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:86158 5.e 5:7248efcd:::benchmark_data_smithi082_106363_object86157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:15.176807+0000", 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 153.97055855799999, 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.062638504999999997, 2023-10-09T04:30:49.232 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 86158 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176807+0000", 2023-10-09T04:30:49.233 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176807+0000", 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176805+0000", 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5882000000000003e-05 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.234 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176881+0000", 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7312999999999998e-05 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176898+0000", 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8850000000000001e-06 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.235 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176905+0000", 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8041000000000003e-05 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.176953+0000", 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000129672 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.177083+0000", 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000159658 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.236 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.177243+0000", 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0063767629999999997 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.183619+0000", 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003651049 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.237 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.187270+0000", 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.052016184 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.239287+0000", 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000117939 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.239404+0000", 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1063000000000003e-05 2023-10-09T04:30:49.238 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:15.239446+0000", 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.062638504999999997 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87233 5.e 5:7fc2a908:::benchmark_data_smithi082_106363_object87232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 152.87942892999999, 2023-10-09T04:30:49.239 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.111294265, 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87233 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.240 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267935+0000", 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2663000000000001e-05 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.241 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267977+0000", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5570000000000001e-06 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267987+0000", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5379999999999996e-06 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267991+0000", 2023-10-09T04:30:49.242 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2032000000000003e-05 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268073+0000", 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000133812 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268207+0000", 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012234499999999999 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.243 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268330+0000", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0025285910000000002 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.270858+0000", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003702388 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.274561+0000", 2023-10-09T04:30:49.244 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10452908399999999 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379090+0000", 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7010999999999997e-05 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379187+0000", 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4277999999999999e-05 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.245 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379231+0000", 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.111294265 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87250 5.e 5:7cc8be50:::benchmark_data_smithi082_106363_object87249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 152.85983584300001, 2023-10-09T04:30:49.246 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12868468, 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87250 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.247 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287527+0000", 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3634999999999996e-05 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.248 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287591+0000", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6143000000000001e-05 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287607+0000", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1359999999999999e-06 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287614+0000", 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0812e-05 2023-10-09T04:30:49.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287675+0000", 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000141256 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287816+0000", 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000197651 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:30:49.250 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.288014+0000", 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031527289999999999 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.291167+0000", 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001236439 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.292403+0000", 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.123718986 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.251 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416122+0000", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4619e-05 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416177+0000", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7647000000000001e-05 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416214+0000", 2023-10-09T04:30:49.252 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12868468 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:105799 5.e 5:7bdf13d4:::benchmark_data_smithi082_106363_object105798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:36.732659+0000", 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 132.41470674300001, 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076042488000000005, 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.253 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 105799 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732659+0000", 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.254 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732659+0000", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999948 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732654+0000", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2364000000000002e-05 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732706+0000", 2023-10-09T04:30:49.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4079e-05 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732720+0000", 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0989999999999999e-06 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732728+0000", 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012881599999999999 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.256 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732857+0000", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012876000000000001 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.732986+0000", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014966099999999999 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.733135+0000", 2023-10-09T04:30:49.257 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029687509999999999 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.736104+0000", 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.072127054999999995 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.808231+0000", 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019338999999999999 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.258 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.808425+0000", 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00027668899999999999 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:36.808701+0000", 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076042488000000005 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.259 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:110538 5.e 5:7fd1eb92:::benchmark_data_smithi082_106363_object110537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:42.052821+0000", 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 127.094544513, 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083329512999999994, 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110538 2023-10-09T04:30:49.260 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052821+0000", 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052821+0000", 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.261 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052819+0000", 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100505 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052920+0000", 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3217000000000001e-05 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.262 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052933+0000", 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1850000000000001e-06 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052939+0000", 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9833999999999998e-05 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.052989+0000", 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000109477 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.263 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053098+0000", 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012319300000000001 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053222+0000", 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049305750000000004 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.264 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.058152+0000", 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.077734406000000006 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.135887+0000", 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2912999999999996e-05 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.135979+0000", 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017112900000000001 2023-10-09T04:30:49.265 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.136151+0000", 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083329512999999994 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:110540 5.e 5:7429fc2a:::benchmark_data_smithi082_106363_object110539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:42.053245+0000", 2023-10-09T04:30:49.266 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 127.094120586, 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083131569000000002, 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110540 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053245+0000", 2023-10-09T04:30:49.267 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053245+0000", 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053244+0000", 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7175000000000001e-05 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.268 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053301+0000", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0393e-05 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053312+0000", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8470000000000001e-06 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.053317+0000", 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026503410000000001 2023-10-09T04:30:49.269 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.055968+0000", 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000116086 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.056084+0000", 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012487199999999999 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.270 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.056209+0000", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002136368 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.058345+0000", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.077885802000000004 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.136231+0000", 2023-10-09T04:30:49.271 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8976e-05 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.136280+0000", 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6676999999999996e-05 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:42.136377+0000", 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083131569000000002 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.272 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:113334 5.e 5:7def54cb:::benchmark_data_smithi082_106363_object113333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:44.997314+0000", 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 124.15005185299999, 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.065824626999999997, 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 113334 2023-10-09T04:30:49.273 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997314+0000", 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997314+0000", 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999971 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.274 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997311+0000", 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3883999999999998e-05 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997395+0000", 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4825999999999999e-05 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.275 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997420+0000", 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2405999999999999e-05 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.997432+0000", 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0038885759999999999 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.001320+0000", 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014687599999999999 2023-10-09T04:30:49.276 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.001467+0000", 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017620299999999999 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.001644+0000", 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0077635940000000004 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.277 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009407+0000", 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.053616076999999998 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063023+0000", 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2166000000000002e-05 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.278 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063075+0000", 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2957e-05 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063138+0000", 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.065824626999999997 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:113336 5.e 5:74e23b75:::benchmark_data_smithi082_106363_object113335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.279 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:44.998763+0000", 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 124.148602224, 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.064493774000000004, 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 113336 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.280 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998763+0000", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998763+0000", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999952 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998759+0000", 2023-10-09T04:30:49.281 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7644000000000003e-05 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998846+0000", 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9425999999999999e-05 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998876+0000", 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1339999999999998e-06 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.282 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:44.998883+0000", 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.006760177 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.005643+0000", 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000172805 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.283 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.005816+0000", 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019102999999999999 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.006007+0000", 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.006867822 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.012875+0000", 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.050297274000000003 2023-10-09T04:30:49.284 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063172+0000", 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1676e-05 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063204+0000", 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3638999999999999e-05 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.285 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.063257+0000", 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.064493774000000004 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:113346 5.e 5:74a4e50e:::benchmark_data_smithi082_106363_object113345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:45.009539+0000", 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 124.13782676300001, 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.064718211999999997, 2023-10-09T04:30:49.286 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 113346 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009539+0000", 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.287 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009539+0000", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999948 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009534+0000", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0486999999999999e-05 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009594+0000", 2023-10-09T04:30:49.288 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5665999999999999e-05 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009610+0000", 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4569999999999999e-06 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009617+0000", 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6934999999999997e-05 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.289 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009664+0000", 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000101133 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009765+0000", 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111935 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0", 2023-10-09T04:30:49.290 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.009877+0000", 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031796680000000001 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.013057+0000", 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.061024244999999998 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.074081+0000", 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4082000000000001e-05 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.291 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.074145+0000", 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111697 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:45.074257+0000", 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.064718211999999997 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:30:49.292 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:30:49.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:49.323 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:49.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:49.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:49.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:49.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:49.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:49.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:49.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:49.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:49.849 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:49 smithi082 bash[77063]: cluster 2023-10-09T04:30:49.479788+0000 mon.a (mon.0) 726 : cluster 3 Health check update: Degraded data redundancy: 16871/76656 objects degraded (22.009%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:49.849 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:49 smithi082 bash[77063]: audit 2023-10-09T04:30:49.532788+0000 mon.a (mon.0) 727 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:49.850 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:49] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.33.4" 2023-10-09T04:30:49.850 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:49 smithi082 bash[79733]: cluster 2023-10-09T04:30:49.479788+0000 mon.a (mon.0) 726 : cluster 3 Health check update: Degraded data redundancy: 16871/76656 objects degraded (22.009%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:49.850 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:49 smithi082 bash[79733]: audit 2023-10-09T04:30:49.532788+0000 mon.a (mon.0) 727 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:49.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:49.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:49.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:49 smithi153 bash[59565]: cluster 2023-10-09T04:30:49.479788+0000 mon.a (mon.0) 726 : cluster 3 Health check update: Degraded data redundancy: 16871/76656 objects degraded (22.009%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:49.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:49 smithi153 bash[59565]: audit 2023-10-09T04:30:49.532788+0000 mon.a (mon.0) 727 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:49.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:30:49.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:50.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:50.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:30:50.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:50.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:50.149 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:50.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:50.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:50.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:50.428 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:50.532 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:50.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:50.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:50.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:50.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:50 smithi082 bash[77063]: cluster 2023-10-09T04:30:49.440146+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v447: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15603/69171 objects degraded (22.557%); 2824/69171 objects misplaced (4.083%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:50 smithi082 bash[79733]: cluster 2023-10-09T04:30:49.440146+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v447: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15603/69171 objects degraded (22.557%); 2824/69171 objects misplaced (4.083%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:50.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:50.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:50 smithi153 bash[59565]: cluster 2023-10-09T04:30:49.440146+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v447: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15603/69171 objects degraded (22.557%); 2824/69171 objects misplaced (4.083%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:50.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:50.924 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:50.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:50.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:51.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:51.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:51.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:51.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:51.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:51.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:51.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:51.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:51.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:51.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:51.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:51.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:51.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:51.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:52.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:52.060 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:52.169 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:52.193 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:52.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:52.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:52.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:52.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:52.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:52.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:52.618 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:52 smithi082 bash[77063]: cluster 2023-10-09T04:30:51.441082+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v448: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14904/65583 objects degraded (22.725%); 2824/65583 objects misplaced (4.306%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:52.618 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:52 smithi082 bash[79733]: cluster 2023-10-09T04:30:51.441082+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v448: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14904/65583 objects degraded (22.725%); 2824/65583 objects misplaced (4.306%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:52.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:52.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:52.860 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:52 smithi153 bash[59565]: cluster 2023-10-09T04:30:51.441082+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v448: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 14904/65583 objects degraded (22.725%); 2824/65583 objects misplaced (4.306%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:30:52.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:52.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:52.945 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:52.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:53.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:53.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:53.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:53.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:53.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:53.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:53.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:53.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:53.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:53.666 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:53.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:53.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:53.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:53.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:53.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:53.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:54.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:54.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:54.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:54.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:54.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:54.334 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:54.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:54.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:54.454 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:54.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:54.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:54.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:54.621 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:54 smithi082 bash[77063]: cluster 2023-10-09T04:30:53.442486+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v449: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 13885/60228 objects degraded (23.054%); 2382/60228 objects misplaced (3.955%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:54.621 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:54 smithi082 bash[77063]: cluster 2023-10-09T04:30:54.480939+0000 mon.a (mon.0) 728 : cluster 3 Health check update: Degraded data redundancy: 13885/60228 objects degraded (23.054%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:54.635 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:54.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:54.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:54.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:54.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:54 smithi153 bash[59565]: cluster 2023-10-09T04:30:53.442486+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v449: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 13885/60228 objects degraded (23.054%); 2382/60228 objects misplaced (3.955%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:54.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:54 smithi153 bash[59565]: cluster 2023-10-09T04:30:54.480939+0000 mon.a (mon.0) 728 : cluster 3 Health check update: Degraded data redundancy: 13885/60228 objects degraded (23.054%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:54.923 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:54.936 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:54 smithi082 bash[79733]: cluster 2023-10-09T04:30:53.442486+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v449: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 13885/60228 objects degraded (23.054%); 2382/60228 objects misplaced (3.955%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:54.936 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:54 smithi082 bash[79733]: cluster 2023-10-09T04:30:54.480939+0000 mon.a (mon.0) 728 : cluster 3 Health check update: Degraded data redundancy: 13885/60228 objects degraded (23.054%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:30:54.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:54.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:54.981 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:54.984 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:55.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:55.097 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:55.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:55.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:55.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:55.374 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:55.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:55.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:55.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:55.678 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:55.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:55.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:55.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:55.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:55.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:56.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.108 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:56.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:56.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:56.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.360 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:56.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:56.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:56.494 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:56 smithi153 bash[59565]: cluster 2023-10-09T04:30:55.444040+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v450: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:56.494 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:56 smithi153 bash[59565]: audit 2023-10-09T04:30:55.459147+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.495 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:56 smithi153 bash[59565]: audit 2023-10-09T04:30:55.467787+0000 mon.a (mon.0) 730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.495 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:56 smithi153 bash[59565]: audit 2023-10-09T04:30:56.004293+0000 mon.a (mon.0) 731 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.495 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:56 smithi153 bash[59565]: audit 2023-10-09T04:30:56.017427+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:56.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[77063]: cluster 2023-10-09T04:30:55.444040+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v450: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[77063]: audit 2023-10-09T04:30:55.459147+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[77063]: audit 2023-10-09T04:30:55.467787+0000 mon.a (mon.0) 730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[77063]: audit 2023-10-09T04:30:56.004293+0000 mon.a (mon.0) 731 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[77063]: audit 2023-10-09T04:30:56.017427+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[79733]: cluster 2023-10-09T04:30:55.444040+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v450: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[79733]: audit 2023-10-09T04:30:55.459147+0000 mon.a (mon.0) 729 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.640 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[79733]: audit 2023-10-09T04:30:55.467787+0000 mon.a (mon.0) 730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[79733]: audit 2023-10-09T04:30:56.004293+0000 mon.a (mon.0) 731 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.641 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:56 smithi082 bash[79733]: audit 2023-10-09T04:30:56.017427+0000 mon.a (mon.0) 732 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:56.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:56.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:56.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:56.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:56.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:56.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:57.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:57.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:57.162 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:57.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:57.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:57.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:57.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:57.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:57.553 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:57.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:57 smithi153 bash[59565]: audit 2023-10-09T04:30:56.599956+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:57.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:57 smithi153 bash[59565]: audit 2023-10-09T04:30:56.601518+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:57.561 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:57 smithi153 bash[59565]: audit 2023-10-09T04:30:56.614955+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:57.580 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:57.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[77063]: audit 2023-10-09T04:30:56.599956+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[77063]: audit 2023-10-09T04:30:56.601518+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[77063]: audit 2023-10-09T04:30:56.614955+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[79733]: audit 2023-10-09T04:30:56.599956+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[79733]: audit 2023-10-09T04:30:56.601518+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:57.744 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:57 smithi082 bash[79733]: audit 2023-10-09T04:30:56.614955+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:30:57.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:57.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:57.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:57.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:57.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:58.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:58.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:58.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:58.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:58.245 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:30:58.245 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:30:58.245 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T04:30:58.245 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:30:58.283 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:58.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:58.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:58.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:30:58 smithi153 bash[59565]: cluster 2023-10-09T04:30:57.444856+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v451: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:58.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:58.688 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:30:58 smithi082 bash[77063]: cluster 2023-10-09T04:30:57.444856+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v451: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:58.689 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:30:58 smithi082 bash[79733]: cluster 2023-10-09T04:30:57.444856+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v451: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 12986/54885 objects degraded (23.660%); 2382/54885 objects misplaced (4.340%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:30:58.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:58.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:58.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:58.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:58.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.067 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:30:59.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:59.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:59.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:59.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:59.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:59.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:59.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:59.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:59.499 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:59.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:59.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:59.748 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:30:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:30:59] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T04:30:59.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:59.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:30:59.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:00.023 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:00.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:00.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:00.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:00.284 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:00.314 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:00.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:00.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[77063]: audit 2023-10-09T04:30:59.267634+0000 mon.a (mon.0) 736 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:00.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[77063]: audit 2023-10-09T04:30:59.269136+0000 mon.a (mon.0) 737 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:59.445893+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v452: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1003 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 11823/47991 objects degraded (24.636%); 1944/47991 objects misplaced (4.051%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[77063]: cluster 2023-10-09T04:30:59.484048+0000 mon.a (mon.0) 738 : cluster 3 Health check update: Degraded data redundancy: 12986/54885 objects degraded (23.660%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[79733]: audit 2023-10-09T04:30:59.267634+0000 mon.a (mon.0) 736 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[79733]: audit 2023-10-09T04:30:59.269136+0000 mon.a (mon.0) 737 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:59.445893+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v452: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1003 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 11823/47991 objects degraded (24.636%); 1944/47991 objects misplaced (4.051%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:31:00.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:00 smithi082 bash[79733]: cluster 2023-10-09T04:30:59.484048+0000 mon.a (mon.0) 738 : cluster 3 Health check update: Degraded data redundancy: 12986/54885 objects degraded (23.660%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:00.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:00.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:00.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:00.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:00 smithi153 bash[59565]: audit 2023-10-09T04:30:59.267634+0000 mon.a (mon.0) 736 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:00 smithi153 bash[59565]: audit 2023-10-09T04:30:59.269136+0000 mon.a (mon.0) 737 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:59.445893+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v452: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 1003 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 11823/47991 objects degraded (24.636%); 1944/47991 objects misplaced (4.051%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:31:00.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:00 smithi153 bash[59565]: cluster 2023-10-09T04:30:59.484048+0000 mon.a (mon.0) 738 : cluster 3 Health check update: Degraded data redundancy: 12986/54885 objects degraded (23.660%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:00.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:00.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:00.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:00.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:00.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:00.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:01.066 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:01.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:01.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:01.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:01.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:01.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:01.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:01.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:01.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:01.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:01.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:01.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:01.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:02.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:02.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:02.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:02.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:02.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:02.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:02.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:02.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:02.532 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:02.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:02.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:02.667 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:02 smithi153 bash[59565]: cluster 2023-10-09T04:31:01.446595+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v453: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 933 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 11164/44667 objects degraded (24.994%); 1944/44667 objects misplaced (4.352%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:31:02.685 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:02.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:02.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:02.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:02 smithi082 bash[77063]: cluster 2023-10-09T04:31:01.446595+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v453: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 933 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 11164/44667 objects degraded (24.994%); 1944/44667 objects misplaced (4.352%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:31:02.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:02 smithi082 bash[79733]: cluster 2023-10-09T04:31:01.446595+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v453: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 933 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 11164/44667 objects degraded (24.994%); 1944/44667 objects misplaced (4.352%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:31:02.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:02.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:02.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:02.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:03.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:03.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:03.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:03.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:03.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:03.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:03.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:03.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:03.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:03.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:03.872 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:03.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:03.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:03.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:04.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:04.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:04.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:04.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:04.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:04.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:04.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.552 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:04.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:04.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:04.799 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:04 smithi082 bash[77063]: cluster 2023-10-09T04:31:03.447411+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v454: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 814 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 10032/38952 objects degraded (25.755%); 1474/38952 objects misplaced (3.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:31:04.799 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:04 smithi082 bash[77063]: cluster 2023-10-09T04:31:04.484969+0000 mon.a (mon.0) 739 : cluster 3 Health check update: Degraded data redundancy: 10032/38952 objects degraded (25.755%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:04.799 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:04 smithi082 bash[79733]: cluster 2023-10-09T04:31:03.447411+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v454: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 814 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 10032/38952 objects degraded (25.755%); 1474/38952 objects misplaced (3.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:31:04.799 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:04 smithi082 bash[79733]: cluster 2023-10-09T04:31:04.484969+0000 mon.a (mon.0) 739 : cluster 3 Health check update: Degraded data redundancy: 10032/38952 objects degraded (25.755%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:04.817 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:04.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:04 smithi153 bash[59565]: cluster 2023-10-09T04:31:03.447411+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v454: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 814 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 10032/38952 objects degraded (25.755%); 1474/38952 objects misplaced (3.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:31:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:04 smithi153 bash[59565]: cluster 2023-10-09T04:31:04.484969+0000 mon.a (mon.0) 739 : cluster 3 Health check update: Degraded data redundancy: 10032/38952 objects degraded (25.755%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:04.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:04.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:04.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:05.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:05.080 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:05.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:05.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:05.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:05.345 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:05.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:05.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:05.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:05.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:05.718 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:05.752 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:05.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:05.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:05.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:05.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:06.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:06.137 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:06.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:06.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:06.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:06.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:06.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:06.515 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:06 smithi082 bash[79733]: cluster 2023-10-09T04:31:05.448137+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v455: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:06.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:06.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:06.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:06 smithi153 bash[59565]: cluster 2023-10-09T04:31:05.448137+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v455: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:06.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:06 smithi082 bash[77063]: cluster 2023-10-09T04:31:05.448137+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v455: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:06.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:06.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T04:31:06.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:06.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:06.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:06.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:07.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:07.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:07.136 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:31:07.190 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:31:07.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:07.226 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:31:07.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:07.263 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:31:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:07.311 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:31:07.357 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:31:07.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:07.397 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:31:07.397 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.397 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.398 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:07.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:07.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:07.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:07.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:07.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[77063]: audit 2023-10-09T04:31:06.834736+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.82:0/1706750130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[77063]: audit 2023-10-09T04:31:06.835933+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[77063]: audit 2023-10-09T04:31:07.084344+0000 mon.a (mon.0) 741 : audit 0 from='client.? 172.21.15.82:0/1327130514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:07.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[79733]: audit 2023-10-09T04:31:06.834736+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.82:0/1706750130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[79733]: audit 2023-10-09T04:31:06.835933+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:07 smithi082 bash[79733]: audit 2023-10-09T04:31:07.084344+0000 mon.a (mon.0) 741 : audit 0 from='client.? 172.21.15.82:0/1327130514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:07.782 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:07 smithi153 bash[59565]: audit 2023-10-09T04:31:06.834736+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.82:0/1706750130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.782 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:07 smithi153 bash[59565]: audit 2023-10-09T04:31:06.835933+0000 mon.a (mon.0) 740 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T04:31:07.782 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:07 smithi153 bash[59565]: audit 2023-10-09T04:31:07.084344+0000 mon.a (mon.0) 741 : audit 0 from='client.? 172.21.15.82:0/1327130514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:07.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:07.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:07.907 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:07.908 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:07.935 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:07.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:07.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:08.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:08.068 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:08.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:08.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:08.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:08.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:08.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:08.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:08.547 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:31:08.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.611 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:08.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:08.732 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:08 smithi153 bash[59565]: cluster 2023-10-09T04:31:07.448905+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v456: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:08.732 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:08 smithi153 bash[59565]: audit 2023-10-09T04:31:07.542800+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:08.733 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:31:08.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:08.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.859 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:08 smithi082 bash[77063]: cluster 2023-10-09T04:31:07.448905+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v456: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:08.859 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:08 smithi082 bash[77063]: audit 2023-10-09T04:31:07.542800+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:08.860 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:08 smithi082 bash[79733]: cluster 2023-10-09T04:31:07.448905+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v456: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 698 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 9108/33357 objects degraded (27.305%); 1474/33357 objects misplaced (4.419%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:31:08.860 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:08 smithi082 bash[79733]: audit 2023-10-09T04:31:07.542800+0000 mon.a (mon.0) 742 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:08.879 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:08.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:08.986 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:09.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:09.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T04:31:09.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:09.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:09.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:09.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:08.524981+0000 mon.a (mon.0) 743 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:08.540393+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: audit 2023-10-09T04:31:08.541390+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:08.544281+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:08.545532+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:08.545788+0000 mon.a (mon.0) 745 : cluster 0 osdmap e129: 8 total, 7 up, 8 in 2023-10-09T04:31:09.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:09 smithi153 bash[59565]: cluster 2023-10-09T04:31:09.486165+0000 mon.a (mon.0) 746 : cluster 3 Health check update: Degraded data redundancy: 9108/33357 objects degraded (27.305%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:09.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:09.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:09.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:09.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:09.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:09.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:09.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:08.524981+0000 mon.a (mon.0) 743 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:08.540393+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: audit 2023-10-09T04:31:08.541390+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:08.544281+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:08.545532+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:08.545788+0000 mon.a (mon.0) 745 : cluster 0 osdmap e129: 8 total, 7 up, 8 in 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[77063]: cluster 2023-10-09T04:31:09.486165+0000 mon.a (mon.0) 746 : cluster 3 Health check update: Degraded data redundancy: 9108/33357 objects degraded (27.305%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:08.524981+0000 mon.a (mon.0) 743 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:08.540393+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: audit 2023-10-09T04:31:08.541390+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.82:0/1842481775' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:08.544281+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:08.545532+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:08.545788+0000 mon.a (mon.0) 745 : cluster 0 osdmap e129: 8 total, 7 up, 8 in 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[79733]: cluster 2023-10-09T04:31:09.486165+0000 mon.a (mon.0) 746 : cluster 3 Health check update: Degraded data redundancy: 9108/33357 objects degraded (27.305%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:09.983 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:09] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T04:31:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:31:10.007 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:31:10.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:10.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:10.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:10.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:10.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:10.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:10.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:10.616 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:10 smithi082 bash[77063]: cluster 2023-10-09T04:31:09.449927+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v458: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 538 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 7837/25698 objects degraded (30.497%); 1007/25698 objects misplaced (3.919%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:10.617 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:10 smithi082 bash[79733]: cluster 2023-10-09T04:31:09.449927+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v458: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 538 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 7837/25698 objects degraded (30.497%); 1007/25698 objects misplaced (3.919%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:10.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:10.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:10.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:10.753 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:10 smithi153 bash[59565]: cluster 2023-10-09T04:31:09.449927+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v458: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 538 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 7837/25698 objects degraded (30.497%); 1007/25698 objects misplaced (3.919%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:10.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:10.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:10.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:10.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:10.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:10.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:11.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:11.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:11.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:11.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:11.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:11.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:11.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:11.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:11.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:11.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:11.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:11.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:11.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:11.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:11.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:11.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:12.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:12.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:12.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:12.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:12.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:12.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:12.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:12.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:12.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:12.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:12.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:12.645 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:12 smithi153 bash[59565]: cluster 2023-10-09T04:31:11.451569+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v459: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 464 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7147/22152 objects degraded (32.263%); 1007/22152 objects misplaced (4.546%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:12.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:12.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:12.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:12.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:12.829 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:31:12.836 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:12 smithi082 bash[79733]: cluster 2023-10-09T04:31:11.451569+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v459: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 464 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7147/22152 objects degraded (32.263%); 1007/22152 objects misplaced (4.546%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:12.837 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:12 smithi082 bash[77063]: cluster 2023-10-09T04:31:11.451569+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v459: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 464 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 7147/22152 objects degraded (32.263%); 1007/22152 objects misplaced (4.546%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:31:12.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:12.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:12.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.023 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:13.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:13.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:13.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:13.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:13.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:13.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.462 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.531 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:13.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:13.600 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:13.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:13.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:13.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:14.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:14.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:14.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:14.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:14.372 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:31:14.372 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:14.372 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T04:31:14.372 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:31:14.372 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:31:14.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T04:31:14.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:14.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:14.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:14.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:14.547 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[77063]: cluster 2023-10-09T04:31:13.452290+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v460: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 348 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 6077/16575 objects degraded (36.664%); 574/16575 objects misplaced (3.463%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:14.547 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[77063]: audit 2023-10-09T04:31:14.271145+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:14.547 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[77063]: audit 2023-10-09T04:31:14.272472+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:14.547 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[77063]: cluster 2023-10-09T04:31:14.493158+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 6077/16575 objects degraded (36.664%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:14.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:14.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:14.762 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:14.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:14 smithi153 bash[59565]: cluster 2023-10-09T04:31:13.452290+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v460: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 348 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 6077/16575 objects degraded (36.664%); 574/16575 objects misplaced (3.463%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:14.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:14 smithi153 bash[59565]: audit 2023-10-09T04:31:14.271145+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:14.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:14 smithi153 bash[59565]: audit 2023-10-09T04:31:14.272472+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:14.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:14 smithi153 bash[59565]: cluster 2023-10-09T04:31:14.493158+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 6077/16575 objects degraded (36.664%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:14.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:14.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:14.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[79733]: cluster 2023-10-09T04:31:13.452290+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v460: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 348 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 6077/16575 objects degraded (36.664%); 574/16575 objects misplaced (3.463%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:14.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[79733]: audit 2023-10-09T04:31:14.271145+0000 mon.a (mon.0) 747 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:31:14.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[79733]: audit 2023-10-09T04:31:14.272472+0000 mon.a (mon.0) 748 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:14.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:14 smithi082 bash[79733]: cluster 2023-10-09T04:31:14.493158+0000 mon.a (mon.0) 749 : cluster 3 Health check update: Degraded data redundancy: 6077/16575 objects degraded (36.664%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:14.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:14.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:15.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.081 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:15.187 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:15.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:15.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:15.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:15.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:15.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:15.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:15.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:15.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:15.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:15.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:15.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:15.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:15.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:16.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:16.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:16.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:16.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:16.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:16.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:16.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:16.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:16.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:16.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:16.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:16.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:16.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:16 smithi082 bash[77063]: cluster 2023-10-09T04:31:15.453149+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v461: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:16.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:16 smithi082 bash[79733]: cluster 2023-10-09T04:31:15.453149+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v461: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:16.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:16.825 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:16 smithi153 bash[59565]: cluster 2023-10-09T04:31:15.453149+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v461: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:16.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:16.870 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:16.871 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:16.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:16.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:16.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:17.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:17.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:17.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:17.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:17.251 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:17.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:17.395 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:17.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:17.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:17.618 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:17.618 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:17.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:17.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:17.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:17.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:17.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:17.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:17.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:17.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:18.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:18.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:18.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:18.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:18.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:18.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:18.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:18.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:18.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:18.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:18.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:18.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:18.669 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:18 smithi153 bash[59565]: cluster 2023-10-09T04:31:17.454097+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v462: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:18.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:31:18.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:18 smithi082 bash[77063]: cluster 2023-10-09T04:31:17.454097+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v462: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:18.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:18 smithi082 bash[79733]: cluster 2023-10-09T04:31:17.454097+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v462: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 57 active+clean; 232 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5159/11022 objects degraded (46.806%); 574/11022 objects misplaced (5.208%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:31:18.736 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:18.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:18.826 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 114228 objects 2023-10-09T04:31:18.826 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :152.856 2023-10-09T04:31:18.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:18.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:18.909 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:18.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:19.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:19.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:19.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:19.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:19.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:19.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:19.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:19.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:19.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:19.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:19 smithi153 bash[59565]: cluster 2023-10-09T04:31:19.494649+0000 mon.a (mon.0) 750 : cluster 3 Health check update: Degraded data redundancy: 5159/11022 objects degraded (46.806%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:19.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:19.944 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:19.951 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T04:31:19.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T04:31:19.953 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:19 smithi082 bash[77063]: cluster 2023-10-09T04:31:19.494649+0000 mon.a (mon.0) 750 : cluster 3 Health check update: Degraded data redundancy: 5159/11022 objects degraded (46.806%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:19.953 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:19] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T04:31:19.953 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:19 smithi082 bash[79733]: cluster 2023-10-09T04:31:19.494649+0000 mon.a (mon.0) 750 : cluster 3 Health check update: Degraded data redundancy: 5159/11022 objects degraded (46.806%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:19.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:19.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:20.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:20.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:20.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:20.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:20.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:20.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:20.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:20.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:20.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:20.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:20.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:20 smithi082 bash[77063]: cluster 2023-10-09T04:31:19.455665+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v463: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 14 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 57 active+clean; 83 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 3952/3831 objects degraded (103.158%); 123/3831 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:31:20.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:20 smithi082 bash[79733]: cluster 2023-10-09T04:31:19.455665+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v463: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 14 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 57 active+clean; 83 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 3952/3831 objects degraded (103.158%); 123/3831 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:31:20.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:20.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:20.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:20 smithi153 bash[59565]: cluster 2023-10-09T04:31:19.455665+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v463: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 14 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 57 active+clean; 83 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 3952/3831 objects degraded (103.158%); 123/3831 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:31:20.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:20.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:20.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:21.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:21.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:21.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:21.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:21.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:21.355 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:21.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:21.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:21.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:21.643 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:21.746 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:21.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:21.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:21.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:21.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:22.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:22.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:22.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:22.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:22.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:22.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:22.413 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:22.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:22.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:22.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:22 smithi082 bash[77063]: cluster 2023-10-09T04:31:21.456983+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v464: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 57 active+clean; 31 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 3486/1353 objects degraded (257.650%); 123/1353 objects misplaced (9.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:31:22.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:22 smithi082 bash[79733]: cluster 2023-10-09T04:31:21.456983+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v464: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 57 active+clean; 31 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 3486/1353 objects degraded (257.650%); 123/1353 objects misplaced (9.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:31:22.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:22.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:22.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:22 smithi153 bash[59565]: cluster 2023-10-09T04:31:21.456983+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v464: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 57 active+clean; 31 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 3486/1353 objects degraded (257.650%); 123/1353 objects misplaced (9.091%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:31:22.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:22.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:23.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:23.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:23.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:23.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:23.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:23.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:23.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:23.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:23.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:23.493 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:23.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:23.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:23.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:23.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:23.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:23.877 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:23.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:23.980 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:24.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:24.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:24.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:24.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:24.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:24.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:24.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:24.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:24.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:24.651 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:24 smithi082 bash[79733]: cluster 2023-10-09T04:31:23.457758+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v465: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3252/84 objects degraded (3871.429%); 1.5 MiB/s, 50 objects/s recovering 2023-10-09T04:31:24.651 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:24 smithi082 bash[79733]: cluster 2023-10-09T04:31:24.495997+0000 mon.a (mon.0) 751 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 13 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:24.651 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:24 smithi082 bash[77063]: cluster 2023-10-09T04:31:23.457758+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v465: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3252/84 objects degraded (3871.429%); 1.5 MiB/s, 50 objects/s recovering 2023-10-09T04:31:24.651 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:24 smithi082 bash[77063]: cluster 2023-10-09T04:31:24.495997+0000 mon.a (mon.0) 751 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 13 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:24.664 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:24.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:24.865 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:24 smithi153 bash[59565]: cluster 2023-10-09T04:31:23.457758+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v465: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3252/84 objects degraded (3871.429%); 1.5 MiB/s, 50 objects/s recovering 2023-10-09T04:31:24.865 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:24 smithi153 bash[59565]: cluster 2023-10-09T04:31:24.495997+0000 mon.a (mon.0) 751 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 13 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:24.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:24.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:25.030 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:25.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:25.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:25.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:25.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:25.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:25.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:25.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.722 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:25.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:25.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:25.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:25.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:25.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:26.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:26.113 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:26.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:26.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:26.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:26.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:26.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:26.582 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:26 smithi082 bash[79733]: cluster 2023-10-09T04:31:25.459124+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v466: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:26.583 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:26 smithi082 bash[77063]: cluster 2023-10-09T04:31:25.459124+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v466: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:26.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:26.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:26.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.754 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:26 smithi153 bash[59565]: cluster 2023-10-09T04:31:25.459124+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v466: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:26.769 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:26.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:26.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:26.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:26.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:27.003 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:27.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:27.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:27.112 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:27.113 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:27.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:27.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:27.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:27.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:27.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:27.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:27.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:27.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:27.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:27.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:27.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:27.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:27.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:27.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:27.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:31:27.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:27.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:28.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:28.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:28.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:28.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:28.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:28.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:28.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:28.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:28.563 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:28.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:28.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:28.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:28.801 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:28 smithi082 bash[77063]: cluster 2023-10-09T04:31:27.460266+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v467: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:28.802 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:28 smithi082 bash[79733]: cluster 2023-10-09T04:31:27.460266+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v467: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:28.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:28.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:28.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:28 smithi153 bash[59565]: cluster 2023-10-09T04:31:27.460266+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v467: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+clean+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 39 objects/s recovering 2023-10-09T04:31:28.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:28.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:28.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:29.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:29.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:29.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.521 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:29.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:29.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.898 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:29 smithi153 bash[59565]: audit 2023-10-09T04:31:29.262978+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:29.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:29 smithi082 bash[79733]: audit 2023-10-09T04:31:29.262978+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:29 smithi082 bash[77063]: audit 2023-10-09T04:31:29.262978+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:29] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T04:31:30.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:30.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:30.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:30.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:30.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:30.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:30.409 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:30.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:30.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:30.564 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:30.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:30.674 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:30 smithi082 bash[77063]: cluster 2023-10-09T04:31:29.461056+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v468: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 76 objects/s recovering 2023-10-09T04:31:30.674 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:30 smithi082 bash[77063]: cluster 2023-10-09T04:31:29.584634+0000 mon.a (mon.0) 753 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:30.675 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:30 smithi082 bash[79733]: cluster 2023-10-09T04:31:29.461056+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v468: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 76 objects/s recovering 2023-10-09T04:31:30.675 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:30 smithi082 bash[79733]: cluster 2023-10-09T04:31:29.584634+0000 mon.a (mon.0) 753 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:30.691 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:30.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:30.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:30.903 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:31:30.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:30.925 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:31:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:30 smithi153 bash[59565]: cluster 2023-10-09T04:31:29.461056+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v468: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 3252/84 objects degraded (3871.429%); 832 KiB/s, 76 objects/s recovering 2023-10-09T04:31:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:30 smithi153 bash[59565]: cluster 2023-10-09T04:31:29.584634+0000 mon.a (mon.0) 753 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:30.945 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:31:30.949 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:30.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:30.968 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:31:30.998 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:31:31.024 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:31:31.050 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:31:31.050 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.050 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.051 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:31.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:31.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:31.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:31.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:31.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:31.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:31.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:31.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:31.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:31.595 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:31 smithi082 bash[77063]: audit 2023-10-09T04:31:30.880258+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.82:0/3066571297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:31.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:31.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:31.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:31.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:31.843 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:31 smithi153 bash[59565]: audit 2023-10-09T04:31:30.880258+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.82:0/3066571297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:31.860 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:31.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:31 smithi082 bash[79733]: audit 2023-10-09T04:31:30.880258+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.82:0/3066571297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:31.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:31.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:32.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:32.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.266 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:32.369 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:31:32.369 INFO:teuthology.orchestra.run.smithi082.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:31:32.369 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:31:32.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:32.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:32.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:32.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:32.603 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:31:32.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:32.642 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:32.710 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:32.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:32.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:32.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:32 smithi082 bash[77063]: cluster 2023-10-09T04:31:31.461828+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v469: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 3252/84 objects degraded (3871.429%); 149 KiB/s, 65 objects/s recovering 2023-10-09T04:31:32.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:32 smithi082 bash[77063]: audit 2023-10-09T04:31:32.364805+0000 mon.a (mon.0) 754 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:32.766 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:32 smithi082 bash[79733]: cluster 2023-10-09T04:31:31.461828+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v469: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 3252/84 objects degraded (3871.429%); 149 KiB/s, 65 objects/s recovering 2023-10-09T04:31:32.766 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:32 smithi082 bash[79733]: audit 2023-10-09T04:31:32.364805+0000 mon.a (mon.0) 754 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:32.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:32.888 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:32.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:32.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:32 smithi153 bash[59565]: cluster 2023-10-09T04:31:31.461828+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v469: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 10 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 3252/84 objects degraded (3871.429%); 149 KiB/s, 65 objects/s recovering 2023-10-09T04:31:32.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:32 smithi153 bash[59565]: audit 2023-10-09T04:31:32.364805+0000 mon.a (mon.0) 754 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:32.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:33.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:33.126 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:33.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:33.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:33.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:33.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:31:33.634 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:31:33.643 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:33.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:33.796 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: cluster 2023-10-09T04:31:32.593435+0000 mon.a (mon.0) 755 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:33.796 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: audit 2023-10-09T04:31:32.598895+0000 mon.a (mon.0) 756 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:33.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: cluster 2023-10-09T04:31:32.598952+0000 mon.a (mon.0) 757 : cluster 0 osdmap e130: 8 total, 7 up, 8 in 2023-10-09T04:31:33.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: cluster 2023-10-09T04:31:33.166912+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: cluster 2023-10-09T04:31:33.167955+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:33 smithi153 bash[59565]: cluster 2023-10-09T04:31:33.168994+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: cluster 2023-10-09T04:31:32.593435+0000 mon.a (mon.0) 755 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:33.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: audit 2023-10-09T04:31:32.598895+0000 mon.a (mon.0) 756 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:33.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: cluster 2023-10-09T04:31:32.598952+0000 mon.a (mon.0) 757 : cluster 0 osdmap e130: 8 total, 7 up, 8 in 2023-10-09T04:31:33.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: cluster 2023-10-09T04:31:33.166912+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: cluster 2023-10-09T04:31:33.167955+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[77063]: cluster 2023-10-09T04:31:33.168994+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: cluster 2023-10-09T04:31:32.593435+0000 mon.a (mon.0) 755 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: audit 2023-10-09T04:31:32.598895+0000 mon.a (mon.0) 756 : audit 1 from='client.? 172.21.15.82:0/3411596106' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: cluster 2023-10-09T04:31:32.598952+0000 mon.a (mon.0) 757 : cluster 0 osdmap e130: 8 total, 7 up, 8 in 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: cluster 2023-10-09T04:31:33.166912+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: cluster 2023-10-09T04:31:33.167955+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.905 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:33 smithi082 bash[79733]: cluster 2023-10-09T04:31:33.168994+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T04:31:33.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:33.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:34.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:34.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:34.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:34.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:34.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:34.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:34.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:34.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:34.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:34.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:34.657 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:34.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:34.687 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:34 smithi082 bash[77063]: cluster 2023-10-09T04:31:33.462903+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:34.687 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:34 smithi082 bash[79733]: cluster 2023-10-09T04:31:33.462903+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:34.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:34.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:34.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:34.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:34 smithi153 bash[59565]: cluster 2023-10-09T04:31:33.462903+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v471: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:34.950 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:34.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:35.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:35.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:35.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:35.316 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:35.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:35.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:35.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:35.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:35.593 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:35.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:35.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:35.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:35.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:35.839 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:35 smithi153 bash[59565]: cluster 2023-10-09T04:31:35.609172+0000 mon.a (mon.0) 758 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 8 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:35.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:35.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.962 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:35 smithi082 bash[77063]: cluster 2023-10-09T04:31:35.609172+0000 mon.a (mon.0) 758 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 8 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:35.963 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:35 smithi082 bash[79733]: cluster 2023-10-09T04:31:35.609172+0000 mon.a (mon.0) 758 : cluster 3 Health check update: Degraded data redundancy: 3252/84 objects degraded (3871.429%), 8 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:35.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:35.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:36.071 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:31:36.071 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:31:36.071 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:31:36.071 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:31:36.071 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:31:36.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:36.099 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:36.202 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:36.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:36.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:36.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:36.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:36.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:36.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:36.750 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:36.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:31:36.854 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:36.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:36 smithi153 bash[59565]: cluster 2023-10-09T04:31:35.463744+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v472: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:36.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:36 smithi082 bash[77063]: cluster 2023-10-09T04:31:35.463744+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v472: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:36 smithi082 bash[79733]: cluster 2023-10-09T04:31:35.463744+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v472: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:37.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:37.110 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:37.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:37.243 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:37.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:37.370 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:37.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:37.503 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:37.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:37.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:37.664 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:37.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:37.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:37.902 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:37.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:37.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:38.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:38.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:38.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:38.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:38.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:38.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:38.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:38.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:38.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:38.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:31:38.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:38.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:38.688 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_0' removed 2023-10-09T04:31:38.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:38.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:38.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:38 smithi153 bash[59565]: cluster 2023-10-09T04:31:37.464375+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v473: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:38.861 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:38 smithi153 bash[59565]: audit 2023-10-09T04:31:38.447449+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:38.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:38.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:38.880 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:38 smithi082 bash[79733]: cluster 2023-10-09T04:31:37.464375+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v473: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:38.880 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:38 smithi082 bash[79733]: audit 2023-10-09T04:31:38.447449+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:38.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:38 smithi082 bash[77063]: cluster 2023-10-09T04:31:37.464375+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v473: 89 pgs: 1 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+undersized, 57 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3252/84 objects degraded (3871.429%); 0 B/s, 91 objects/s recovering 2023-10-09T04:31:38.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:38 smithi082 bash[77063]: audit 2023-10-09T04:31:38.447449+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:38.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:39.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:39.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:39.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:39.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:39.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:39.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:39.776 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:31:39.777 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:31:39.777 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T04:31:39.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T04:31:39.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:39 smithi153 bash[59565]: cluster 2023-10-09T04:31:38.678506+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:39 smithi153 bash[59565]: audit 2023-10-09T04:31:38.681677+0000 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:39 smithi153 bash[59565]: cluster 2023-10-09T04:31:38.681748+0000 mon.a (mon.0) 761 : cluster 0 osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:31:39.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:39 smithi153 bash[59565]: cluster 2023-10-09T04:31:39.260416+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:39.969 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:39] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-09T04:31:39.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[79733]: cluster 2023-10-09T04:31:38.678506+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[79733]: audit 2023-10-09T04:31:38.681677+0000 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:39.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[79733]: cluster 2023-10-09T04:31:38.681748+0000 mon.a (mon.0) 761 : cluster 0 osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:31:39.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[79733]: cluster 2023-10-09T04:31:39.260416+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[77063]: cluster 2023-10-09T04:31:38.678506+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[77063]: audit 2023-10-09T04:31:38.681677+0000 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.82:0/2103310192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:39.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[77063]: cluster 2023-10-09T04:31:38.681748+0000 mon.a (mon.0) 761 : cluster 0 osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:31:39.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:39 smithi082 bash[77063]: cluster 2023-10-09T04:31:39.260416+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T04:31:39.987 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:39.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:40.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:40.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:40.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:40.252 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:40.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:40.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:40.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:40.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:40.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:40.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:40.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:40.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:40.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:40.911 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:40 smithi082 bash[77063]: cluster 2023-10-09T04:31:39.465478+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v475: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:40.911 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:40 smithi082 bash[79733]: cluster 2023-10-09T04:31:39.465478+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v475: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:40.929 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:41.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:41.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:41.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:41.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:40 smithi153 bash[59565]: cluster 2023-10-09T04:31:39.465478+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v475: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:41.208 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:41.268 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:41.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:41.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:41.418 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:31:41.418 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:41.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:41.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:41.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:41.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:41.591 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:41.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:41.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:41.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:41.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:41.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:41.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:41.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:41.977 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:42.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:42.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:42.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:42.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:42.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:42.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:42.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:42.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:42.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:42.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:42.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.673 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:42.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.790 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:42 smithi082 bash[79733]: cluster 2023-10-09T04:31:41.466714+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v476: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:42.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:42 smithi082 bash[77063]: cluster 2023-10-09T04:31:41.466714+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v476: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:42.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:42.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:42.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:42.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:42.942 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:42 smithi153 bash[59565]: cluster 2023-10-09T04:31:41.466714+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v476: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:43.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:43.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:43.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:43.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:43.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:43.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:43.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:43.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:43.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:43.852 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:43.873 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:43.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:43.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:44.002 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.029 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:44.132 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:44.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:44.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:44.316 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:44.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:44.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:44.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:44.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:44.647 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:44.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.793 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:44 smithi153 bash[59565]: cluster 2023-10-09T04:31:43.468051+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v477: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:44.793 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:44 smithi153 bash[59565]: audit 2023-10-09T04:31:44.262668+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.793 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:44 smithi153 bash[59565]: cluster 2023-10-09T04:31:44.503488+0000 mon.a (mon.0) 763 : cluster 3 Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:44.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:44.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:44.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:44.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:44.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:44.952 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[77063]: cluster 2023-10-09T04:31:43.468051+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v477: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:44.952 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[77063]: audit 2023-10-09T04:31:44.262668+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.953 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[77063]: cluster 2023-10-09T04:31:44.503488+0000 mon.a (mon.0) 763 : cluster 3 Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:44.953 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[79733]: cluster 2023-10-09T04:31:43.468051+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v477: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:44.953 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[79733]: audit 2023-10-09T04:31:44.262668+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.953 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:44 smithi082 bash[79733]: cluster 2023-10-09T04:31:44.503488+0000 mon.a (mon.0) 763 : cluster 3 Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:31:45.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:45.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:45.092 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:31:45.092 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:31:45.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:31:45.092 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:31:45.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:15934 5.4 5:2ec4130b:::benchmark_data_smithi082_106363_object15933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:21.938903+0000", 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 263.146188704, 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083077891000000001, 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 15934 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938903+0000", 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938903+0000", 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938900+0000", 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9703999999999998e-05 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938940+0000", 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2751999999999999e-05 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938953+0000", 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.571e-06 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.938960+0000", 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7318999999999999e-05 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939008+0000", 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015746900000000001 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939165+0000", 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00023267000000000001 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939398+0000", 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020126509999999998 2023-10-09T04:31:45.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941410+0000", 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0054320640000000003 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.946842+0000", 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074966626999999994 2023-10-09T04:31:45.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.021809+0000", 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0196000000000002e-05 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.021899+0000", 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1194999999999998e-05 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.021980+0000", 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083077891000000001 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:15936 5.4 5:22992b0b:::benchmark_data_smithi082_106363_object15935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:21.939423+0000", 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 263.14566869100003, 2023-10-09T04:31:45.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.082810393999999996, 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 15936 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939423+0000", 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939423+0000", 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939421+0000", 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9535000000000004e-05 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939491+0000", 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2119e-05 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939503+0000", 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5259999999999999e-06 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.939510+0000", 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00078605299999999999 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.940296+0000", 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013456199999999999 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.940430+0000", 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025888400000000002 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.940689+0000", 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001805294 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.942494+0000", 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0045084790000000001 2023-10-09T04:31:45.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.947003+0000", 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.075065713000000006 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.022069+0000", 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6515999999999996e-05 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.022155+0000", 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7831999999999996e-05 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.022233+0000", 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.082810393999999996 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:15944 5.4 5:2ab99d20:::benchmark_data_smithi082_106363_object15943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:21.941437+0000", 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 263.14365400899999, 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084394010000000005, 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.108 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 15944 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941437+0000", 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941437+0000", 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:45.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941435+0000", 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.29e-05 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941468+0000", 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3998e-05 2023-10-09T04:31:45.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941482+0000", 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6250000000000001e-06 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941489+0000", 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0901e-05 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941529+0000", 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000129973 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941659+0000", 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016595 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.941825+0000", 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0017962709999999999 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.943622+0000", 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035139310000000001 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.947136+0000", 2023-10-09T04:31:45.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.078199675999999996 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.025335+0000", 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00028803500000000002 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.025623+0000", 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020797099999999999 2023-10-09T04:31:45.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.025831+0000", 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084394010000000005 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:15963 5.4 5:2fec674d:::benchmark_data_smithi082_106363_object15962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.115 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:21.949818+0000", 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 263.13527282699999, 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076782242000000001, 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 15963 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.116 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949818+0000", 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949818+0000", 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949817+0000", 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2611999999999997e-05 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949870+0000", 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7583000000000001e-05 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949887+0000", 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:31:45.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949895+0000", 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7512000000000003e-05 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.949933+0000", 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9308999999999995e-05 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.950022+0000", 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000109582 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.950132+0000", 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001046263 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.120 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.951178+0000", 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00077705899999999997 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:21.951955+0000", 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074064521999999994 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.026019+0000", 2023-10-09T04:31:45.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000271539 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.026291+0000", 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00030971500000000001 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:22.026601+0000", 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076782242000000001 2023-10-09T04:31:45.122 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20158 5.4 5:26a77d84:::benchmark_data_smithi082_106363_object20157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.856217+0000", 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 261.22887419, 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12598786200000001, 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.123 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20158 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856217+0000", 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.124 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856217+0000", 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999914 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856208+0000", 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026294300000000002 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.125 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856471+0000", 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7961999999999998e-05 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856489+0000", 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0010000000000004e-06 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856496+0000", 2023-10-09T04:31:45.126 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1356000000000001e-05 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856548+0000", 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000139204 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856687+0000", 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000170273 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.127 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856857+0000", 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015424049999999999 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.858400+0000", 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0041491660000000001 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.128 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.862549+0000", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.119531078 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982080+0000", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2665999999999998e-05 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982152+0000", 2023-10-09T04:31:45.129 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2555000000000001e-05 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982205+0000", 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12598786200000001 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.130 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.131 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20162 5.4 5:27f1fb29:::benchmark_data_smithi082_106363_object20161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.131 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.856746+0000", 2023-10-09T04:31:45.131 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 261.22834480400002, 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12598294099999999, 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20162 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.132 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856746+0000", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856746+0000", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856746+0000", 2023-10-09T04:31:45.133 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0927999999999996e-05 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856817+0000", 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6559999999999998e-06 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856825+0000", 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3980000000000001e-06 2023-10-09T04:31:45.134 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.856830+0000", 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00092409499999999999 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.857754+0000", 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3920999999999997e-05 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.135 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.857848+0000", 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000157054 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.858005+0000", 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023143700000000001 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.136 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860319+0000", 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0042046150000000001 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.864524+0000", 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.117972626 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982496+0000", 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013098600000000001 2023-10-09T04:31:45.137 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982627+0000", 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000102193 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.982729+0000", 2023-10-09T04:31:45.138 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12598294099999999 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20178 5.4 5:26f09c2f:::benchmark_data_smithi082_106363_object20177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.865650+0000", 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 261.21944150500002, 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.117714898, 2023-10-09T04:31:45.139 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20178 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865650+0000", 2023-10-09T04:31:45.140 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865650+0000", 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865648+0000", 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7228999999999998e-05 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.141 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865695+0000", 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7241e-05 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865712+0000", 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0880000000000001e-06 2023-10-09T04:31:45.142 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865721+0000", 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4737999999999997e-05 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.865776+0000", 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000383769 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.143 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.866160+0000", 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026461300000000002 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.866424+0000", 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023415150000000002 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.144 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.868766+0000", 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047842099999999998 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869244+0000", 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.113978595 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.983223+0000", 2023-10-09T04:31:45.145 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7666999999999998e-05 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.983321+0000", 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4168999999999997e-05 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.983365+0000", 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.117714898 2023-10-09T04:31:45.146 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:31937 5.4 5:226afad6:::benchmark_data_smithi082_106363_object31936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:30.492623+0000", 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 254.59246804599999, 2023-10-09T04:31:45.147 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076514118000000006, 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 31937 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.148 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492623+0000", 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492623+0000", 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492621+0000", 2023-10-09T04:31:45.149 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9295000000000004e-05 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492721+0000", 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9463999999999999e-05 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492740+0000", 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.815e-06 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.150 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.492747+0000", 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0052764099999999996 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.498023+0000", 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016245699999999999 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.151 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.498186+0000", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019802900000000001 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.498384+0000", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0021530249999999998 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.500537+0000", 2023-10-09T04:31:45.152 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0053442020000000002 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.505881+0000", 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.063171852000000001 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.569053+0000", 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1446999999999998e-05 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.153 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.569094+0000", 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2966999999999997e-05 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:30.569137+0000", 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076514118000000006 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.154 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:37430 5.4 5:203fd81d:::benchmark_data_smithi082_106363_object37429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:33.962332+0000", 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.12275972099999, 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.073976131000000001, 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.155 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 37430 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.157 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962332+0000", 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962332+0000", 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.158 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962329+0000", 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8318000000000006e-05 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962398+0000", 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7524999999999999e-05 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962415+0000", 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9469999999999992e-06 2023-10-09T04:31:45.159 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962424+0000", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4484999999999998e-05 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962479+0000", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015787099999999999 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962636+0000", 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018797599999999999 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.962824+0000", 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001357203 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.964182+0000", 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015562659999999999 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.161 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:33.965738+0000", 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.070199681 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.035938+0000", 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026296199999999999 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.162 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.036201+0000", 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000107144 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.036308+0000", 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.073976131000000001 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.163 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:38624 5.4 5:207acc77:::benchmark_data_smithi082_106363_object38623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 250.27752768299999, 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183254056, 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 38624 2023-10-09T04:31:45.164 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.165 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807562+0000", 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0506999999999997e-05 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807602+0000", 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7394e-05 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.166 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807620+0000", 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7710000000000006e-06 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807628+0000", 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9972999999999997e-05 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807677+0000", 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111838 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.167 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807789+0000", 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100987 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807890+0000", 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00161356 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.168 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.809504+0000", 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023769960000000001 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.811881+0000", 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.178780785 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.169 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990662+0000", 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0154000000000005e-05 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990752+0000", 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5858999999999994e-05 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990818+0000", 2023-10-09T04:31:45.170 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183254056 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71665 5.4 5:2450f5bb:::benchmark_data_smithi082_106363_object71664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.211788+0000", 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 223.873303378, 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.101252566, 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.171 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71665 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211788+0000", 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.172 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211788+0000", 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211786+0000", 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5839e-05 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.173 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211822+0000", 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8359000000000002e-05 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211840+0000", 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3170000000000004e-06 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.174 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211848+0000", 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7042999999999997e-05 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211895+0000", 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018764899999999999 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.175 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.212082+0000", 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016217099999999999 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.212244+0000", 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0028824179999999999 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.215127+0000", 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0067465579999999997 2023-10-09T04:31:45.176 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.221873+0000", 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090900542000000001 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.312774+0000", 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015470499999999999 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.177 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.312929+0000", 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111768 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.313040+0000", 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.101252566 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.178 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71668 5.4 5:29944daa:::benchmark_data_smithi082_106363_object71667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.216843+0000", 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 223.86824812699999, 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.102164987, 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71668 2023-10-09T04:31:45.179 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216843+0000", 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216843+0000", 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.180 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216842+0000", 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8970000000000001e-05 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216880+0000", 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4865000000000001e-05 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.181 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216895+0000", 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0530000000000001e-06 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216902+0000", 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0639000000000001e-05 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216953+0000", 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000148259 2023-10-09T04:31:45.182 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.217101+0000", 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018367899999999999 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.217285+0000", 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029278910000000002 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.183 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.220213+0000", 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018218119999999999 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.222035+0000", 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.096870764999999998 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.318905+0000", 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0432e-05 2023-10-09T04:31:45.184 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.318966+0000", 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2264000000000001e-05 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.319008+0000", 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.102164987 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.185 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:83157 5.4 5:2310f28c:::benchmark_data_smithi082_106363_object83156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:12.065953+0000", 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 213.01913820999999, 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10704775499999999, 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.186 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 83157 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.065953+0000", 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.065953+0000", 2023-10-09T04:31:45.187 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.065952+0000", 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8066999999999995e-05 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066050+0000", 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2847000000000001e-05 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.188 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066063+0000", 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1689999999999997e-06 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066069+0000", 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2369e-05 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.189 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066111+0000", 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016844399999999999 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066279+0000", 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019452099999999999 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.066474+0000", 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031372470000000001 2023-10-09T04:31:45.190 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.069611+0000", 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022216419999999998 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.071833+0000", 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.100970875 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.191 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.172804+0000", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015422400000000001 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.172958+0000", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2837000000000003e-05 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.173001+0000", 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10704775499999999 2023-10-09T04:31:45.192 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:83169 5.4 5:200029aa:::benchmark_data_smithi082_106363_object83168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:12.078804+0000", 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 213.00628689499999, 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.094385898999999995, 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.193 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 83169 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078804+0000", 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.194 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078804+0000", 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078802+0000", 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5911000000000001e-05 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078848+0000", 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5125000000000001e-05 2023-10-09T04:31:45.195 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078873+0000", 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0672e-05 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078884+0000", 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3302999999999999e-05 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.196 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.078957+0000", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014838500000000001 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.079105+0000", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016700800000000001 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.079272+0000", 2023-10-09T04:31:45.197 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002710755 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.081983+0000", 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016896599999999999 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.082152+0000", 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090928037000000003 2023-10-09T04:31:45.198 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.173080+0000", 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2487e-05 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.173143+0000", 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7759000000000001e-05 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.199 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:12.173190+0000", 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.094385898999999995 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87241 5.4 5:2c0c9c59:::benchmark_data_smithi082_106363_object87240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.272759+0000", 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 208.81233264700001, 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10704978599999999, 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.200 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87241 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272759+0000", 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.201 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272759+0000", 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272758+0000", 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9184999999999995e-05 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.202 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272837+0000", 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8949999999999993e-06 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272847+0000", 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1340000000000003e-06 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.203 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272852+0000", 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3684999999999999e-05 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.272896+0000", 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014932699999999999 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.204 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.273045+0000", 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000169643 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.273215+0000", 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.010267521999999999 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.283482+0000", 2023-10-09T04:31:45.205 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020708900000000001 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.283689+0000", 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095340697000000002 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379030+0000", 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00065594899999999996 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.206 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379686+0000", 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012246499999999999 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379808+0000", 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10704978599999999 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.207 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87249 5.4 5:2139f6e8:::benchmark_data_smithi082_106363_object87248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.287415+0000", 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 208.79767611299999, 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.102890616, 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.208 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87249 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287415+0000", 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287415+0000", 2023-10-09T04:31:45.209 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287414+0000", 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2985999999999998e-05 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287457+0000", 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0098999999999999e-05 2023-10-09T04:31:45.210 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287467+0000", 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0560000000000004e-06 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287472+0000", 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5210999999999999e-05 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.211 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287507+0000", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000184484 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287692+0000", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021247 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287904+0000", 2023-10-09T04:31:45.212 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0032448109999999998 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.291149+0000", 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0047774239999999997 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.295926+0000", 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.094289116000000006 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.213 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.390215+0000", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2003999999999999e-05 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.390267+0000", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8308e-05 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.390306+0000", 2023-10-09T04:31:45.214 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.102890616 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:115643 5.4 5:2f7baa07:::benchmark_data_smithi082_106363_object1412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:47.401070+0000", 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.68402117100001, 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.080863719000000001, 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.215 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 115643 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401070+0000", 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.216 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401070+0000", 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401069+0000", 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5169999999999999e-06 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.217 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401075+0000", 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6410000000000008e-06 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401083+0000", 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3510000000000004e-06 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401088+0000", 2023-10-09T04:31:45.218 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014255299999999999 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401230+0000", 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015200000000000001 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401382+0000", 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103838 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.219 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2", 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.401486+0000", 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0063215220000000004 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.407807+0000", 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.073596643000000003 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.220 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.481404+0000", 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000172743 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.481577+0000", 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00035696799999999999 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.481934+0000", 2023-10-09T04:31:45.221 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.080863719000000001 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:115646 5.4 5:2073128a:::benchmark_data_smithi082_106363_object1415:head [delete] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:47.402065+0000", 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.683026708, 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.080168590999999997, 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.222 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 115646 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402065+0000", 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.223 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402065+0000", 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402064+0000", 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3440000000000001e-06 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.224 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402072+0000", 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9619999999999999e-06 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402081+0000", 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5580000000000002e-06 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.402086+0000", 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022478670000000002 2023-10-09T04:31:45.225 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.404333+0000", 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5731999999999998e-05 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.404419+0000", 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1018000000000002e-05 2023-10-09T04:31:45.226 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2", 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.404510+0000", 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033011519999999999 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.407811+0000", 2023-10-09T04:31:45.227 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074217317000000005 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.482029+0000", 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.517e-05 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.482104+0000", 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012931499999999999 2023-10-09T04:31:45.228 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:47.482233+0000", 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.080168590999999997 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:149466 5.4 5:25c291d1:::benchmark_data_smithi082_106363_object35235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T04:31:45.229 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:29:31.048243+0000", 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 134.03684826599999, 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10585093, 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 149466 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.230 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048243+0000", 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048243+0000", 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.231 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048242+0000", 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1990000000000006e-06 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048250+0000", 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3468e-05 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.232 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048263+0000", 2023-10-09T04:31:45.233 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9800000000000001e-06 2023-10-09T04:31:45.248 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.248 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.248 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.248 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048270+0000", 2023-10-09T04:31:45.248 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2021999999999998e-05 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048312+0000", 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100615 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048413+0000", 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7553999999999995e-05 2023-10-09T04:31:45.249 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2", 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.048510+0000", 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034904010000000002 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.052001+0000", 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.101997295 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.250 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.153998+0000", 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5513999999999999e-05 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.154034+0000", 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0278999999999997e-05 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.251 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:29:31.154094+0000", 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10585093 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:184371 5.4 5:2c2f5776:::benchmark_data_smithi082_106363_object70140:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:30:18.501899+0000", 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 86.583192409000006, 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.094468153999999999, 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:31:45.252 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 184371 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501899+0000", 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.253 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501899+0000", 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501897+0000", 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1226000000000001e-05 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:31:45.254 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501909+0000", 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3594000000000001e-05 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501922+0000", 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6380000000000004e-06 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501930+0000", 2023-10-09T04:31:45.255 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0309000000000002e-05 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.501980+0000", 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000157629 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.502138+0000", 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014530700000000001 2023-10-09T04:31:45.256 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2", 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.502283+0000", 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0094958750000000008 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.511779+0000", 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084419114000000003 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.257 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.596198+0000", 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1722000000000003e-05 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.596260+0000", 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000107132 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:31:45.258 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:30:18.596367+0000", 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.094468153999999999 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:31:45.259 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:31:45.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:45.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:45.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:45.518 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:45.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:45.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:45.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:45.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:31:45.911 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:45.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:45.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:46.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:46.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:46.128 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:31:46.150 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:31:46.170 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:31:46.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:46.192 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:31:46.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:46.215 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:31:46.239 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.262 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:31:46.263 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.263 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:46.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:46.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:46.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:46.464 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:46.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:46.698 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:46 smithi082 bash[77063]: cluster 2023-10-09T04:31:45.468836+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v478: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:46.725 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:46 smithi082 bash[77063]: audit 2023-10-09T04:31:46.102739+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.82:0/4249099554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:46.726 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:46 smithi082 bash[79733]: cluster 2023-10-09T04:31:45.468836+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v478: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:46.726 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:46 smithi082 bash[79733]: audit 2023-10-09T04:31:46.102739+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.82:0/4249099554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:46.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:46.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:46.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:46.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:47.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:47.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:46 smithi153 bash[59565]: cluster 2023-10-09T04:31:45.468836+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v478: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:46 smithi153 bash[59565]: audit 2023-10-09T04:31:46.102739+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.82:0/4249099554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:47.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:47.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:47.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:47.385 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:47.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:47.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:47.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:47.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:47.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:47.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:47.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:47.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:47.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:48.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:48.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:48.280 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:48.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:48.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:48.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:48.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:48.525 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:31:48.525 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:31:48.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:48.582 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:48.686 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:48.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:48.796 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:48 smithi082 bash[77063]: cluster 2023-10-09T04:31:47.469504+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v479: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:48.797 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:48 smithi082 bash[79733]: cluster 2023-10-09T04:31:47.469504+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v479: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:48.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:48.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:48 smithi153 bash[59565]: cluster 2023-10-09T04:31:47.469504+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v479: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:48.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:48.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:49.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:49.098 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:49.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:49.202 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:49.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:49.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:49.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:49.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:49.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:49.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:49.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:49.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:49.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:49.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:49.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:49.870 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:49] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.33.4" 2023-10-09T04:31:49.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:49.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:49.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:50.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:50.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:50.270 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:50.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:50.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:50.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:50.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:50.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:50.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:50.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:50.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:50.776 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:50 smithi082 bash[77063]: cluster 2023-10-09T04:31:49.470706+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v480: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:50.776 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:50 smithi082 bash[79733]: cluster 2023-10-09T04:31:49.470706+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v480: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:50.792 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:50.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:50.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:50.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:51.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:51.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:51.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:51.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:50 smithi153 bash[59565]: cluster 2023-10-09T04:31:49.470706+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v480: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:51.193 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:51.297 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:51.327 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:51.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:51.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:51.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:51.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:51.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:51.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:51.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:51.737 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_1' created 2023-10-09T04:31:51.745 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:31:51.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:51.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:51 smithi153 bash[59565]: audit 2023-10-09T04:31:51.051937+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.82:0/3610840962' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:51 smithi153 bash[59565]: audit 2023-10-09T04:31:51.052544+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:51 smithi153 bash[59565]: audit 2023-10-09T04:31:51.053428+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.82:0/350457620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:51.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:51 smithi153 bash[59565]: audit 2023-10-09T04:31:51.494788+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:51.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:51.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:51.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[79733]: audit 2023-10-09T04:31:51.051937+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.82:0/3610840962' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[79733]: audit 2023-10-09T04:31:51.052544+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[79733]: audit 2023-10-09T04:31:51.053428+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.82:0/350457620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[79733]: audit 2023-10-09T04:31:51.494788+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:51.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[77063]: audit 2023-10-09T04:31:51.051937+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.82:0/3610840962' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[77063]: audit 2023-10-09T04:31:51.052544+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:31:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[77063]: audit 2023-10-09T04:31:51.053428+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.82:0/350457620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:51 smithi082 bash[77063]: audit 2023-10-09T04:31:51.494788+0000 mon.a (mon.0) 765 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:52.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:52.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:52.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:52.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:52.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:52.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:52.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:52.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:52.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.471850+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v481: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.707877+0000 mon.a (mon.0) 766 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: audit 2023-10-09T04:31:51.724149+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: audit 2023-10-09T04:31:51.724267+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.724336+0000 mon.a (mon.0) 769 : cluster 0 osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:31:52.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.724792+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.726289+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.728690+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[77063]: cluster 2023-10-09T04:31:51.728747+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.471850+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v481: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.707877+0000 mon.a (mon.0) 766 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: audit 2023-10-09T04:31:51.724149+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: audit 2023-10-09T04:31:51.724267+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.724336+0000 mon.a (mon.0) 769 : cluster 0 osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.724792+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.726289+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.728690+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:52 smithi082 bash[79733]: cluster 2023-10-09T04:31:51.728747+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:52.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:52.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:52.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:52.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:53.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:53.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:53.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.471850+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v481: 73 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 14 active+undersized, 51 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.707877+0000 mon.a (mon.0) 766 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: audit 2023-10-09T04:31:51.724149+0000 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: audit 2023-10-09T04:31:51.724267+0000 mon.a (mon.0) 768 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.724336+0000 mon.a (mon.0) 769 : cluster 0 osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.724792+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.726289+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:53.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.728690+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:53.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:52 smithi153 bash[59565]: cluster 2023-10-09T04:31:51.728747+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T04:31:53.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:53.275 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:53.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:53.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:53.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T04:31:53.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:53.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:53.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:53.600 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:53.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:53.720 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:53.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:53.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:53.878 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:53.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:53.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:53.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[77063]: cluster 2023-10-09T04:31:52.726328+0000 mon.a (mon.0) 770 : cluster 0 osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[77063]: cluster 2023-10-09T04:31:52.727231+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[77063]: cluster 2023-10-09T04:31:52.731897+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[77063]: cluster 2023-10-09T04:31:52.731924+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[77063]: cluster 2023-10-09T04:31:52.731972+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[79733]: cluster 2023-10-09T04:31:52.726328+0000 mon.a (mon.0) 770 : cluster 0 osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[79733]: cluster 2023-10-09T04:31:52.727231+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[79733]: cluster 2023-10-09T04:31:52.731897+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[79733]: cluster 2023-10-09T04:31:52.731924+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:53 smithi082 bash[79733]: cluster 2023-10-09T04:31:52.731972+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:53 smithi153 bash[59565]: cluster 2023-10-09T04:31:52.726328+0000 mon.a (mon.0) 770 : cluster 0 osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:31:53.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:53 smithi153 bash[59565]: cluster 2023-10-09T04:31:52.727231+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:53 smithi153 bash[59565]: cluster 2023-10-09T04:31:52.731897+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:53 smithi153 bash[59565]: cluster 2023-10-09T04:31:52.731924+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:53.995 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:53 smithi153 bash[59565]: cluster 2023-10-09T04:31:52.731972+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T04:31:54.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:54.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:54.120 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:54.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:54.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:54.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:54.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:54.394 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:54.525 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:54.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:54.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:54.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:54.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:53.472790+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v484: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 12 unknown, 7 active+undersized+degraded, 14 active+undersized, 53 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:53.736110+0000 mon.a (mon.0) 771 : cluster 0 osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.166561+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.169100+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.171610+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.369635+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:54 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.372553+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:54.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:54.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:55.004 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:53.472790+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v484: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 12 unknown, 7 active+undersized+degraded, 14 active+undersized, 53 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:53.736110+0000 mon.a (mon.0) 771 : cluster 0 osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.166561+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.169100+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.171610+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.369635+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.372553+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:53.472790+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v484: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 12 unknown, 7 active+undersized+degraded, 14 active+undersized, 53 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:55.005 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:53.736110+0000 mon.a (mon.0) 771 : cluster 0 osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:31:55.006 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.166561+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.006 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.169100+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.006 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.171610+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.006 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.369635+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.006 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:54 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.372553+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:55.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:55.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:55.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:55.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:31:55.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:55.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:55.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:55.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:55.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:55.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:55.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:55.906 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:56.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:56.022 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:55 smithi082 bash[77063]: cluster 2023-10-09T04:31:54.397748+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:56.022 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:55 smithi082 bash[79733]: cluster 2023-10-09T04:31:54.397748+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:56.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:56.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.153 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:55 smithi153 bash[59565]: cluster 2023-10-09T04:31:54.397748+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T04:31:56.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:56.270 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:56.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:56.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:56.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:56.576 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:56.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:56.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:56.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:56.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:56.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:56.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:56.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:57.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:57.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:56 smithi153 bash[59565]: cluster 2023-10-09T04:31:55.473491+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v486: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:56 smithi153 bash[59565]: audit 2023-10-09T04:31:56.703862+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:57.195 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:57.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:57.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:56 smithi082 bash[77063]: cluster 2023-10-09T04:31:55.473491+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v486: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:57.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:56 smithi082 bash[77063]: audit 2023-10-09T04:31:56.703862+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:57.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:56 smithi082 bash[79733]: cluster 2023-10-09T04:31:55.473491+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v486: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:57.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:56 smithi082 bash[79733]: audit 2023-10-09T04:31:56.703862+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:57.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:57.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:57.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:57.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:57.626 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:57.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.732 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:57.732 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:57.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:57.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:57.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:57.895 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:57.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:58.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.144 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:58.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:58.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:58.381 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:58.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:58.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:58.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:58.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:58.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:58.785 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:58 smithi153 bash[59565]: cluster 2023-10-09T04:31:57.474394+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v487: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:58.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:58.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:58.915 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:58 smithi082 bash[77063]: cluster 2023-10-09T04:31:57.474394+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v487: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:58.915 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:58 smithi082 bash[79733]: cluster 2023-10-09T04:31:57.474394+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v487: 89 pgs: 2 creating+peering, 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 60 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:31:58.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:58.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:59.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:59.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:59.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:59.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:59.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:59.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:59.553 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:59.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:59.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:31:59.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:59.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:59.829 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:31:59 smithi153 bash[59565]: audit 2023-10-09T04:31:59.263906+0000 mon.a (mon.0) 773 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:59.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:59.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:31:59 smithi082 bash[77063]: audit 2023-10-09T04:31:59.263906+0000 mon.a (mon.0) 773 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.958 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:31:59 smithi082 bash[79733]: audit 2023-10-09T04:31:59.263906+0000 mon.a (mon.0) 773 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.959 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:31:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:31:59] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.33.4" 2023-10-09T04:31:59.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:59.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:00.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:00.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:00.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:00.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.374 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:00.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:00.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:00.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:00.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:00.735 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:00.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:00.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:01.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:01.019 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:00 smithi153 bash[59565]: cluster 2023-10-09T04:31:59.475150+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:01.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:01.142 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.146 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:00 smithi082 bash[77063]: cluster 2023-10-09T04:31:59.475150+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:01.146 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:00 smithi082 bash[79733]: cluster 2023-10-09T04:31:59.475150+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v488: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:01.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:01.173 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:01.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:01.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:01.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:01.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:01.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:01.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:01.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:01.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:01.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:01.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:01.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:02.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:02.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:02.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:02.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:02.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:02.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:02.493 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:02.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:02.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:02.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:02.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:02.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:02 smithi082 bash[79733]: cluster 2023-10-09T04:32:01.475841+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:02 smithi082 bash[77063]: cluster 2023-10-09T04:32:01.475841+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:02.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:02.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:03.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:02 smithi153 bash[59565]: cluster 2023-10-09T04:32:01.475841+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v489: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:03.049 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:03.149 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:32:03.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:03.172 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:32:03.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:03.194 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:32:03.223 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:32:03.248 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:32:03.273 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:32:03.286 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.295 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:32:03.295 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.295 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.296 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:32:03.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:03.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:03.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:03.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:03.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:03.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:03.807 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:03.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:03.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:03.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:03.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:04.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:04.102 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:04.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:04.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:04.252 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[79733]: audit 2023-10-09T04:32:03.042961+0000 mon.a (mon.0) 774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.252 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[79733]: audit 2023-10-09T04:32:03.049272+0000 mon.a (mon.0) 775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.252 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[79733]: audit 2023-10-09T04:32:03.122780+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.82:0/4281627841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.252 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[79733]: cluster 2023-10-09T04:32:03.476923+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v490: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:04.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[77063]: audit 2023-10-09T04:32:03.042961+0000 mon.a (mon.0) 774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[77063]: audit 2023-10-09T04:32:03.049272+0000 mon.a (mon.0) 775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[77063]: audit 2023-10-09T04:32:03.122780+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.82:0/4281627841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.253 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:04 smithi082 bash[77063]: cluster 2023-10-09T04:32:03.476923+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v490: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:04.333 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:04.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:04 smithi153 bash[59565]: audit 2023-10-09T04:32:03.042961+0000 mon.a (mon.0) 774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:04 smithi153 bash[59565]: audit 2023-10-09T04:32:03.049272+0000 mon.a (mon.0) 775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:04 smithi153 bash[59565]: audit 2023-10-09T04:32:03.122780+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.82:0/4281627841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:04 smithi153 bash[59565]: cluster 2023-10-09T04:32:03.476923+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v490: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:04.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:04.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:04.597 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:04.627 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:04.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:04.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:04.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:04.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:04.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:04.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:05.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:05.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:05.149 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:05.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:05.287 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:05.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:05.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:05.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:05.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:05.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:05.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:05 smithi153 bash[59565]: audit 2023-10-09T04:32:04.597345+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:05 smithi153 bash[59565]: audit 2023-10-09T04:32:04.605258+0000 mon.a (mon.0) 777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:05 smithi153 bash[59565]: audit 2023-10-09T04:32:05.022066+0000 mon.a (mon.0) 778 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:05 smithi153 bash[59565]: audit 2023-10-09T04:32:05.023073+0000 mon.a (mon.0) 779 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:05 smithi153 bash[59565]: audit 2023-10-09T04:32:05.030367+0000 mon.a (mon.0) 780 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.679 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.681 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[77063]: audit 2023-10-09T04:32:04.597345+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.681 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[77063]: audit 2023-10-09T04:32:04.605258+0000 mon.a (mon.0) 777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.681 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[77063]: audit 2023-10-09T04:32:05.022066+0000 mon.a (mon.0) 778 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:05.681 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[77063]: audit 2023-10-09T04:32:05.023073+0000 mon.a (mon.0) 779 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:05.682 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[77063]: audit 2023-10-09T04:32:05.030367+0000 mon.a (mon.0) 780 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:05.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[79733]: audit 2023-10-09T04:32:04.597345+0000 mon.a (mon.0) 776 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[79733]: audit 2023-10-09T04:32:04.605258+0000 mon.a (mon.0) 777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[79733]: audit 2023-10-09T04:32:05.022066+0000 mon.a (mon.0) 778 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:05.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[79733]: audit 2023-10-09T04:32:05.023073+0000 mon.a (mon.0) 779 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:05.683 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:05 smithi082 bash[79733]: audit 2023-10-09T04:32:05.030367+0000 mon.a (mon.0) 780 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:05.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:05.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:32:05.896 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:32:05.917 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:05.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:06.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:06.081 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:06.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:06.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:06.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:06.360 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:06.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:06.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:06.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:06.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:06 smithi082 bash[79733]: cluster 2023-10-09T04:32:05.478309+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v491: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:06.712 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:06 smithi082 bash[79733]: audit 2023-10-09T04:32:06.531894+0000 mon.a (mon.0) 781 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:32:06.712 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:06 smithi082 bash[77063]: cluster 2023-10-09T04:32:05.478309+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v491: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:06.713 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:06 smithi082 bash[77063]: audit 2023-10-09T04:32:06.531894+0000 mon.a (mon.0) 781 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:32:06.727 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:06.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:06.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:06.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:06.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:06 smithi153 bash[59565]: cluster 2023-10-09T04:32:05.478309+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v491: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:06.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:06 smithi153 bash[59565]: audit 2023-10-09T04:32:06.531894+0000 mon.a (mon.0) 781 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:32:06.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:06.990 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:06.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:07.039 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T04:32:07.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:07.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:07.218 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:07.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:07.358 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:07.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:07.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:07.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:07.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:07.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:07.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:07.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:07.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:07.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:32:07.924 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:07.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:08.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:08.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:08.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:08.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[79733]: cluster 2023-10-09T04:32:07.032258+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[79733]: audit 2023-10-09T04:32:07.034807+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:32:08.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[79733]: cluster 2023-10-09T04:32:07.034895+0000 mon.a (mon.0) 783 : cluster 0 osdmap e135: 8 total, 7 up, 8 in 2023-10-09T04:32:08.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[79733]: cluster 2023-10-09T04:32:07.310072+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[79733]: cluster 2023-10-09T04:32:07.478874+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v493: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:08.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[77063]: cluster 2023-10-09T04:32:07.032258+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[77063]: audit 2023-10-09T04:32:07.034807+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:32:08.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[77063]: cluster 2023-10-09T04:32:07.034895+0000 mon.a (mon.0) 783 : cluster 0 osdmap e135: 8 total, 7 up, 8 in 2023-10-09T04:32:08.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[77063]: cluster 2023-10-09T04:32:07.310072+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:08 smithi082 bash[77063]: cluster 2023-10-09T04:32:07.478874+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v493: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:08.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:08.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:08.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:08 smithi153 bash[59565]: cluster 2023-10-09T04:32:07.032258+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:08 smithi153 bash[59565]: audit 2023-10-09T04:32:07.034807+0000 mon.a (mon.0) 782 : audit 1 from='client.? 172.21.15.82:0/142199335' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:32:08.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:08 smithi153 bash[59565]: cluster 2023-10-09T04:32:07.034895+0000 mon.a (mon.0) 783 : cluster 0 osdmap e135: 8 total, 7 up, 8 in 2023-10-09T04:32:08.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:08 smithi153 bash[59565]: cluster 2023-10-09T04:32:07.310072+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:08.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:08 smithi153 bash[59565]: cluster 2023-10-09T04:32:07.478874+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v493: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:08.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:08.416 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:08.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:08.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:08.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:08.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:08.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:08.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:08.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:08.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:08.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:08.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:09.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:09.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:09.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:09.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:09 smithi082 bash[77063]: cluster 2023-10-09T04:32:07.310366+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:09.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:09 smithi082 bash[79733]: cluster 2023-10-09T04:32:07.310366+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:09.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:09.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:09.364 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:09 smithi153 bash[59565]: cluster 2023-10-09T04:32:07.310366+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T04:32:09.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:09.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:09.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:09.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:09.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:09.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:09.745 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:09.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:09.774 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T04:32:09.774 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T04:32:09.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:32:09.815 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:32:09.815 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:32:09.850 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:32:09.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:32:09.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_128868 2023-10-09T04:32:09.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:32:09.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:32:09.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:09.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:09.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:10.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:10.039 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:10.081 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:09] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.33.4" 2023-10-09T04:32:10.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:10.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:10.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:10.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:10.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:10.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:10.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:10.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:10.539 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:32:10.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:10.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[77063]: cluster 2023-10-09T04:32:09.479506+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v494: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[77063]: audit 2023-10-09T04:32:09.544051+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.82:0/1621986484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[77063]: audit 2023-10-09T04:32:09.544376+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[79733]: cluster 2023-10-09T04:32:09.479506+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v494: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[79733]: audit 2023-10-09T04:32:09.544051+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.82:0/1621986484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:10 smithi082 bash[79733]: audit 2023-10-09T04:32:09.544376+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:10.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:10.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:10.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:10 smithi153 bash[59565]: cluster 2023-10-09T04:32:09.479506+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v494: 89 pgs: 1 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 62 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 12/84 objects degraded (14.286%) 2023-10-09T04:32:10.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:10 smithi153 bash[59565]: audit 2023-10-09T04:32:09.544051+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.82:0/1621986484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.815 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:10 smithi153 bash[59565]: audit 2023-10-09T04:32:09.544376+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:10.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:10.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 16 3025 3009 188.051 188.062 0.00331773 0.0052987 2023-10-09T04:32:10.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:10.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:10.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:11.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:11.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:11.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:11.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:11.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:11.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:11.481 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:11.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:11.610 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:11.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:11.761 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: cluster 2023-10-09T04:32:10.513650+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:11.761 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: cluster 2023-10-09T04:32:10.525064+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.761 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: audit 2023-10-09T04:32:10.528027+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: cluster 2023-10-09T04:32:10.528113+0000 mon.a (mon.0) 787 : cluster 0 osdmap e136: 8 total, 7 up, 8 in 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: cluster 2023-10-09T04:32:10.533645+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[79733]: cluster 2023-10-09T04:32:10.535071+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: cluster 2023-10-09T04:32:10.513650+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: cluster 2023-10-09T04:32:10.525064+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: audit 2023-10-09T04:32:10.528027+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: cluster 2023-10-09T04:32:10.528113+0000 mon.a (mon.0) 787 : cluster 0 osdmap e136: 8 total, 7 up, 8 in 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: cluster 2023-10-09T04:32:10.533645+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:11 smithi082 bash[77063]: cluster 2023-10-09T04:32:10.535071+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.840 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 16 5994 5978 186.799 185.562 0.00867717 0.00533698 2023-10-09T04:32:11.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:11.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:11.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:11.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: cluster 2023-10-09T04:32:10.513650+0000 mon.a (mon.0) 785 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: cluster 2023-10-09T04:32:10.525064+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: audit 2023-10-09T04:32:10.528027+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: cluster 2023-10-09T04:32:10.528113+0000 mon.a (mon.0) 787 : cluster 0 osdmap e136: 8 total, 7 up, 8 in 2023-10-09T04:32:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: cluster 2023-10-09T04:32:10.533645+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:11 smithi153 bash[59565]: cluster 2023-10-09T04:32:10.535071+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T04:32:11.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:11.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:12.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:12.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:12.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:12.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:12.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:12.503 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:12.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:12.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:12.664 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:12 smithi082 bash[77063]: cluster 2023-10-09T04:32:11.480379+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v496: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 64 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s wr, 117 op/s; 344/2913 objects degraded (11.809%) 2023-10-09T04:32:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:12 smithi082 bash[77063]: cluster 2023-10-09T04:32:11.525262+0000 mon.a (mon.0) 788 : cluster 3 Health check update: Degraded data redundancy: 344/2913 objects degraded (11.809%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:12.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:12 smithi082 bash[79733]: cluster 2023-10-09T04:32:11.480379+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v496: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 64 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s wr, 117 op/s; 344/2913 objects degraded (11.809%) 2023-10-09T04:32:12.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:12 smithi082 bash[79733]: cluster 2023-10-09T04:32:11.525262+0000 mon.a (mon.0) 788 : cluster 3 Health check update: Degraded data redundancy: 344/2913 objects degraded (11.809%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:12.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:12.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 16 8593 8577 178.674 162.438 0.00418945 0.00558891 2023-10-09T04:32:12.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:12.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:12 smithi153 bash[59565]: cluster 2023-10-09T04:32:11.480379+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v496: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 64 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s wr, 117 op/s; 344/2913 objects degraded (11.809%) 2023-10-09T04:32:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:12 smithi153 bash[59565]: cluster 2023-10-09T04:32:11.525262+0000 mon.a (mon.0) 788 : cluster 3 Health check update: Degraded data redundancy: 344/2913 objects degraded (11.809%), 10 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:12.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:12.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:13.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:13.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:13.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:13.193 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:13.297 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:13.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:13.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:13.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:13.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:13.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:13.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:13.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:13.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:13.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:13.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 16 11165 11149 174.187 160.75 0.00456311 0.00573088 2023-10-09T04:32:13.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:13.970 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:13.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:14.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:14.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:14.201 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:32:14.201 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:32:14.201 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:32:14.201 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:32:14.201 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:32:14.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:14.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:14.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:14.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:14.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:14.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:14.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:14.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:14.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:14.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:14.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:14.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[77063]: cluster 2023-10-09T04:32:13.481037+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v497: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 191 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s; 344/9024 objects degraded (3.812%) 2023-10-09T04:32:14.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[77063]: audit 2023-10-09T04:32:14.268355+0000 mon.a (mon.0) 789 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:14.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[77063]: audit 2023-10-09T04:32:14.276224+0000 mon.a (mon.0) 790 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:14.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[79733]: cluster 2023-10-09T04:32:13.481037+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v497: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 191 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s; 344/9024 objects degraded (3.812%) 2023-10-09T04:32:14.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[79733]: audit 2023-10-09T04:32:14.268355+0000 mon.a (mon.0) 789 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:14.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:14 smithi082 bash[79733]: audit 2023-10-09T04:32:14.276224+0000 mon.a (mon.0) 790 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:14.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:14.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 13476 13460 168.234 144.438 0.0041915 0.00593411 2023-10-09T04:32:14.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:14.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:14 smithi153 bash[59565]: cluster 2023-10-09T04:32:13.481037+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v497: 89 pgs: 1 active+clean+remapped, 10 active+undersized+degraded, 16 active+undersized, 62 active+clean; 191 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 372 op/s; 344/9024 objects degraded (3.812%) 2023-10-09T04:32:14.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:14 smithi153 bash[59565]: audit 2023-10-09T04:32:14.268355+0000 mon.a (mon.0) 789 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:14.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:14 smithi153 bash[59565]: audit 2023-10-09T04:32:14.276224+0000 mon.a (mon.0) 790 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:14.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:14.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:14.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:15.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:15.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.279 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.381 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:15.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:15.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:15.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:15.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.668 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:15.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:32:15.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:15.851 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 16 15924 15908 165.693 153 0.00305986 0.0060271 2023-10-09T04:32:15.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:15.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:32:16.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:16.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:16.214 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:16.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:16.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:16.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:16.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:16.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:16.612 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:16.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:16 smithi153 bash[59565]: cluster 2023-10-09T04:32:15.482108+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v498: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:16.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:16.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:16.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 18023 18007 160.761 131.188 0.00281185 0.00621443 2023-10-09T04:32:16.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:16 smithi082 bash[77063]: cluster 2023-10-09T04:32:15.482108+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v498: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:16.890 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:16 smithi082 bash[79733]: cluster 2023-10-09T04:32:15.482108+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v498: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:16.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:16.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:17.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.043 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:17.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:17.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:17.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:17.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:17.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:17.452 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:17.455 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:17.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:17.569 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:17.673 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.704 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:17.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:17.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:17.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 20191 20175 157.602 135.5 0.0099462 0.00633782 2023-10-09T04:32:17.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:17.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:17.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:18.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:18.097 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:18.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:18.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:18.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:18.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:18.461 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:18.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:18.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:18.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:18.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:18 smithi082 bash[77063]: cluster 2023-10-09T04:32:17.485790+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v499: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:18.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:18 smithi082 bash[79733]: cluster 2023-10-09T04:32:17.485790+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v499: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:18.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:18.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:18.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 22327 22311 154.922 133.5 0.00525395 0.00644858 2023-10-09T04:32:18.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:18.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:18 smithi153 bash[59565]: cluster 2023-10-09T04:32:17.485790+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v499: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 794 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 4413/37971 objects degraded (11.622%) 2023-10-09T04:32:18.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:18.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:19.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:19.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:19.144 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:19.280 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:19.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:19.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:19.438 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:19.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:19.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:19.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:19.687 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:19 smithi082 bash[77063]: cluster 2023-10-09T04:32:19.514898+0000 mon.a (mon.0) 791 : cluster 3 Health check update: Degraded data redundancy: 4413/37971 objects degraded (11.622%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:19.687 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:19 smithi082 bash[79733]: cluster 2023-10-09T04:32:19.514898+0000 mon.a (mon.0) 791 : cluster 3 Health check update: Degraded data redundancy: 4413/37971 objects degraded (11.622%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:19.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:19.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:19.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:19 smithi153 bash[59565]: cluster 2023-10-09T04:32:19.514898+0000 mon.a (mon.0) 791 : cluster 3 Health check update: Degraded data redundancy: 4413/37971 objects degraded (11.622%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:19.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:19.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 23755 23739 148.354 89.25 0.00357238 0.0067327 2023-10-09T04:32:19.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:19.950 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:19] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-09T04:32:19.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:19.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:20.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:20.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:20.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:20.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:20.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.358 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:20.461 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:20.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:20.509 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:20.510 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:20.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:20 smithi153 bash[59565]: cluster 2023-10-09T04:32:19.486542+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v500: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 5688/52635 objects degraded (10.806%) 2023-10-09T04:32:20.619 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:20.722 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:20 smithi082 bash[77063]: cluster 2023-10-09T04:32:19.486542+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v500: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 5688/52635 objects degraded (10.806%) 2023-10-09T04:32:20.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:20 smithi082 bash[79733]: cluster 2023-10-09T04:32:19.486542+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v500: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 5688/52635 objects degraded (10.806%) 2023-10-09T04:32:20.750 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:20.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 16 25641 25625 145.581 117.875 0.00182098 0.00681552 2023-10-09T04:32:20.853 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:20.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:20.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:21.011 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:21.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:21.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:21.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:21.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.271 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:21.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:21.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:21.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:21.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.527 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:21.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:21.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:21.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:21.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 27301 27285 142.094 103.75 0.00634111 0.00703117 2023-10-09T04:32:21.894 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:21.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:21.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:22.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:22.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:22.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:22.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:22.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:22.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:22.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:22.439 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:22.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:22.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:22.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:22.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:22.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:22.805 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:22 smithi153 bash[59565]: cluster 2023-10-09T04:32:21.487506+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v501: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s; 7576/69489 objects degraded (10.902%) 2023-10-09T04:32:22.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:22.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 29298 29282 140.764 124.812 0.0108322 0.00709839 2023-10-09T04:32:22.926 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:22.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:22.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:22 smithi082 bash[77063]: cluster 2023-10-09T04:32:21.487506+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v501: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s; 7576/69489 objects degraded (10.902%) 2023-10-09T04:32:22.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:22 smithi082 bash[79733]: cluster 2023-10-09T04:32:21.487506+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v501: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s; 7576/69489 objects degraded (10.902%) 2023-10-09T04:32:23.054 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:23.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:23.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:23.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:23.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:23.299 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:23.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:23.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:23.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:23.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:23.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:23.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:23.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:23.852 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 30952 30936 138.093 103.375 0.0094927 0.00723518 2023-10-09T04:32:23.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:23.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:23.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:24.084 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:24.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:24.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:24.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:24.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:24.370 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:24.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:24.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:24.597 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:24.606 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:24 smithi153 bash[59565]: cluster 2023-10-09T04:32:23.488152+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v502: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 7576/76497 objects degraded (9.904%) 2023-10-09T04:32:24.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:24 smithi153 bash[59565]: cluster 2023-10-09T04:32:24.516299+0000 mon.a (mon.0) 792 : cluster 3 Health check update: Degraded data redundancy: 7576/76497 objects degraded (9.904%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:24.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:24.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:24.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:24.752 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:24.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:24 smithi082 bash[77063]: cluster 2023-10-09T04:32:23.488152+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v502: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 7576/76497 objects degraded (9.904%) 2023-10-09T04:32:24.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:24 smithi082 bash[77063]: cluster 2023-10-09T04:32:24.516299+0000 mon.a (mon.0) 792 : cluster 3 Health check update: Degraded data redundancy: 7576/76497 objects degraded (9.904%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:24.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:24 smithi082 bash[79733]: cluster 2023-10-09T04:32:23.488152+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v502: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 7576/76497 objects degraded (9.904%) 2023-10-09T04:32:24.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:24 smithi082 bash[79733]: cluster 2023-10-09T04:32:24.516299+0000 mon.a (mon.0) 792 : cluster 3 Health check update: Degraded data redundancy: 7576/76497 objects degraded (9.904%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:24.853 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 16 32436 32420 135.069 92.75 0.00843477 0.00739683 2023-10-09T04:32:24.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:24.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:24.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:24.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:25.012 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:25.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:25.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:25.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:25.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:25.283 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:25.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:25.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:25.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:25.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:25.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:25.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:25.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:25.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:25.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:25.853 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 34097 34081 133.115 103.812 0.0096011 0.00750668 2023-10-09T04:32:25.906 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:25.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:25.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:26.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:26.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:26.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.729 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.820 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:26 smithi082 bash[79733]: cluster 2023-10-09T04:32:25.489045+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v503: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:26.820 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:26 smithi082 bash[77063]: cluster 2023-10-09T04:32:25.489045+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v503: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:26.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:26.853 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 16 35761 35745 131.398 104 0.00345967 0.00760299 2023-10-09T04:32:26.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:26.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:26.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:26 smithi153 bash[59565]: cluster 2023-10-09T04:32:25.489045+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v503: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:26.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:26.980 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:26.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:27.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:27.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:27.209 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:27.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:27.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:27.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:27.466 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:27.488 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:27.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:27.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:27.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:27.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:27.853 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 37432 37416 129.9 104.438 0.00682949 0.00769277 2023-10-09T04:32:27.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:27.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:27.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:27.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:28.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:28.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:28.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:28.283 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:28.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:28.386 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:28.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.552 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:28.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:28 smithi153 bash[59565]: cluster 2023-10-09T04:32:27.489640+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v504: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:28.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:28.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.824 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:28.853 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 15 39009 38994 128.253 98.625 0.012175 0.00779144 2023-10-09T04:32:28.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:28 smithi082 bash[79733]: cluster 2023-10-09T04:32:27.489640+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v504: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:28.886 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:28 smithi082 bash[77063]: cluster 2023-10-09T04:32:27.489640+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v504: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s; 10317/95838 objects degraded (10.765%) 2023-10-09T04:32:28.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:28.957 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:28.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:29.062 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:29.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:29.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:29.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:29.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:29.352 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:29.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:29.488 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:29.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:29.591 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:29.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:29.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:29.740 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:29 smithi153 bash[59565]: audit 2023-10-09T04:32:29.263937+0000 mon.a (mon.0) 793 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:29.740 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:29 smithi153 bash[59565]: cluster 2023-10-09T04:32:29.517522+0000 mon.a (mon.0) 794 : cluster 3 Health check update: Degraded data redundancy: 10317/95838 objects degraded (10.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:29.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:29.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:32:29.853417+0000 min lat: 0.00116931 max lat: 0.243754 avg lat: 0.00789215 2023-10-09T04:32:29.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:32:29.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 16 40540 40524 126.622 95.625 0.0057347 0.00789215 2023-10-09T04:32:29.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:29.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:29.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:29.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:29 smithi082 bash[77063]: audit 2023-10-09T04:32:29.263937+0000 mon.a (mon.0) 793 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:29 smithi082 bash[77063]: cluster 2023-10-09T04:32:29.517522+0000 mon.a (mon.0) 794 : cluster 3 Health check update: Degraded data redundancy: 10317/95838 objects degraded (10.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:29] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T04:32:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:29 smithi082 bash[79733]: audit 2023-10-09T04:32:29.263937+0000 mon.a (mon.0) 793 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:29 smithi082 bash[79733]: cluster 2023-10-09T04:32:29.517522+0000 mon.a (mon.0) 794 : cluster 3 Health check update: Degraded data redundancy: 10317/95838 objects degraded (10.765%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:30.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:30.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:30.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:30.155 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:30.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:30.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.287 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:30.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.422 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:30.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:30.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:30.585 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:30 smithi082 bash[77063]: cluster 2023-10-09T04:32:29.491345+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v505: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 11339/106749 objects degraded (10.622%) 2023-10-09T04:32:30.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:30.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:30.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:30.819 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:30.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 42085 42069 125.19 96.5625 0.00367683 0.00798077 2023-10-09T04:32:30.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:30 smithi153 bash[59565]: cluster 2023-10-09T04:32:29.491345+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v505: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 11339/106749 objects degraded (10.622%) 2023-10-09T04:32:30.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:30.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:30.972 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:30 smithi082 bash[79733]: cluster 2023-10-09T04:32:29.491345+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v505: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s; 11339/106749 objects degraded (10.622%) 2023-10-09T04:32:31.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:31.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:31.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:31.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:31.316 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:31.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:31.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:31.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:31.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:31.608 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:31.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:31.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:31.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:31.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:31.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 43673 43657 124.009 99.25 0.00731964 0.00805821 2023-10-09T04:32:31.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:31.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:31.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:32.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:32.119 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:32.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:32.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:32.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:32.416 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:32.520 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:32.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:32.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:32 smithi082 bash[77063]: cluster 2023-10-09T04:32:31.493093+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v506: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 12742/119202 objects degraded (10.689%) 2023-10-09T04:32:32.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:32 smithi082 bash[79733]: cluster 2023-10-09T04:32:31.493093+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v506: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 12742/119202 objects degraded (10.689%) 2023-10-09T04:32:32.786 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:32.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:32.854 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 15 45260 45245 122.932 99.25 0.00584261 0.00812902 2023-10-09T04:32:32.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:32 smithi153 bash[59565]: cluster 2023-10-09T04:32:31.493093+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v506: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 12742/119202 objects degraded (10.689%) 2023-10-09T04:32:32.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:32.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:33.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:33.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:33.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:33.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:33.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:33.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:33.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:33.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:33.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:33.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:33.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:33.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:33.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.857 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 46692 46676 121.522 89.4375 0.0043806 0.00822201 2023-10-09T04:32:33.868 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:33.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:33.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:33.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:34.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:34.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:34.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:34.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:34.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:34.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:34.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:34.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:34.534 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:34.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:34.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.675 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:32:34.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:34.726 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:32:34.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:34.783 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:32:34.787 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:34 smithi153 bash[59565]: cluster 2023-10-09T04:32:33.493858+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v507: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 12742/125325 objects degraded (10.167%) 2023-10-09T04:32:34.787 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:34 smithi153 bash[59565]: cluster 2023-10-09T04:32:34.518618+0000 mon.a (mon.0) 795 : cluster 3 Health check update: Degraded data redundancy: 12742/125325 objects degraded (10.167%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:34.787 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:34 smithi153 bash[59565]: audit 2023-10-09T04:32:34.589317+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.82:0/1844180777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:34.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:34.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:34.822 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:32:34.846 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:34.846 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:34.857 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 47828 47812 119.501 71 0.0156604 0.00836173 2023-10-09T04:32:34.883 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:32:34.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[77063]: cluster 2023-10-09T04:32:33.493858+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v507: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 12742/125325 objects degraded (10.167%) 2023-10-09T04:32:34.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[77063]: cluster 2023-10-09T04:32:34.518618+0000 mon.a (mon.0) 795 : cluster 3 Health check update: Degraded data redundancy: 12742/125325 objects degraded (10.167%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:34.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[77063]: audit 2023-10-09T04:32:34.589317+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.82:0/1844180777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:34.897 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[79733]: cluster 2023-10-09T04:32:33.493858+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v507: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 12742/125325 objects degraded (10.167%) 2023-10-09T04:32:34.897 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[79733]: cluster 2023-10-09T04:32:34.518618+0000 mon.a (mon.0) 795 : cluster 3 Health check update: Degraded data redundancy: 12742/125325 objects degraded (10.167%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:34.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:34 smithi082 bash[79733]: audit 2023-10-09T04:32:34.589317+0000 mon.a (mon.0) 796 : audit 0 from='client.? 172.21.15.82:0/1844180777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:34.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:34.939 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:32:34.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:34.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:34.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:34.997 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:32:34.997 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.997 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:34.998 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:35.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:35.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:35.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:35.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:35.226 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:32:35.239 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:32:35.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:35.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:35.575 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:35.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:35.631 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:32:35.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:35.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:35.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:35.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:35 smithi153 bash[59565]: audit 2023-10-09T04:32:35.058177+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.82:0/3926132954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.852 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:35 smithi153 bash[59565]: audit 2023-10-09T04:32:35.058961+0000 mon.a (mon.0) 797 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.857 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 49164 49148 118.117 83.5 0.00551609 0.00846041 2023-10-09T04:32:35.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:35.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:35.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:35 smithi082 bash[77063]: audit 2023-10-09T04:32:35.058177+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.82:0/3926132954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:35 smithi082 bash[77063]: audit 2023-10-09T04:32:35.058961+0000 mon.a (mon.0) 797 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:35 smithi082 bash[79733]: audit 2023-10-09T04:32:35.058177+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.82:0/3926132954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:35 smithi082 bash[79733]: audit 2023-10-09T04:32:35.058961+0000 mon.a (mon.0) 797 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:35.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:36.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.133 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:36.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:36.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:36.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:36.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:36.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:36.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:36.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:36.678 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:36.779 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: audit 2023-10-09T04:32:35.208428+0000 mgr.y (mgr.34107) 548 : audit 0 from='client.44933 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:32:36.779 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.494751+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v508: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:36.779 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.600272+0000 mon.a (mon.0) 798 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.613740+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.619309+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: audit 2023-10-09T04:32:35.621307+0000 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.621399+0000 mon.a (mon.0) 800 : cluster 0 osdmap e137: 8 total, 7 up, 8 in 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[77063]: cluster 2023-10-09T04:32:35.626631+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: audit 2023-10-09T04:32:35.208428+0000 mgr.y (mgr.34107) 548 : audit 0 from='client.44933 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.494751+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v508: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:36.780 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.600272+0000 mon.a (mon.0) 798 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:36.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.613740+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.619309+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: audit 2023-10-09T04:32:35.621307+0000 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:36.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.621399+0000 mon.a (mon.0) 800 : cluster 0 osdmap e137: 8 total, 7 up, 8 in 2023-10-09T04:32:36.781 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:36 smithi082 bash[79733]: cluster 2023-10-09T04:32:35.626631+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:36.857 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 50543 50527 116.934 86.1875 0.00705229 0.00854642 2023-10-09T04:32:36.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: audit 2023-10-09T04:32:35.208428+0000 mgr.y (mgr.34107) 548 : audit 0 from='client.44933 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.494751+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v508: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.600272+0000 mon.a (mon.0) 798 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.613740+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.619309+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: audit 2023-10-09T04:32:35.621307+0000 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:36.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.621399+0000 mon.a (mon.0) 800 : cluster 0 osdmap e137: 8 total, 7 up, 8 in 2023-10-09T04:32:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:36 smithi153 bash[59565]: cluster 2023-10-09T04:32:35.626631+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T04:32:36.941 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:36.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:37.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:37.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:37.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:37.170 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:37.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:37.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:37.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:37.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:37.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:37.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:37.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:37.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:37.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:37.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:37.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 16 51839 51823 115.645 81 0.0134464 0.00864111 2023-10-09T04:32:37.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:37.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:37.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:37.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:37.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:38.098 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:38.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:38.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:38.252 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:38.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:38.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:38.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:38.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:38.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:38.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:32:38.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:38.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:38.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:38.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 53031 53015 114.226 74.5 0.0151046 0.00874786 2023-10-09T04:32:38.878 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:38 smithi082 bash[77063]: cluster 2023-10-09T04:32:37.495365+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v510: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:38.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:38 smithi082 bash[79733]: cluster 2023-10-09T04:32:37.495365+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v510: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:32:38.880 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:32:38.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:38.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:38 smithi153 bash[59565]: cluster 2023-10-09T04:32:37.495365+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v510: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 15010/141597 objects degraded (10.601%) 2023-10-09T04:32:38.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:38.985 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":507,"stamp":"2023-10-09T04:32:33.493280+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2740721587,"num_objects":41775,"num_object_clones":0,"num_object_copies":125325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41775,"num_whiteouts":0,"num_read":6302,"num_read_kb":13759,"num_write":42853,"num_write_kb":2686967,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42917,"ondisk_log_size":42917,"up":240,"acting":241,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12561808,"kb_used_data":10738000,"kb_used_omap":34,"kb_used_meta":1823709,"kb_avail":737399408,"statfs":{"total":767960285184,"available":755096993792,"internally_reserved":0,"allocated":10995712000,"data_stored":10991140327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":1867478741},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,6,7,12,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1219756032,"num_objects":18612,"num_object_clones":0,"num_object_copies":55836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18612,"num_write_kb":1191168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006495"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317090+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:32:07.317090+0000","last_peered":"2023-10-09T04:32:07.317090+0000","last_clean":"2023-10-09T04:32:07.317090+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:32:07.317090+0000","last_undegraded":"2023-10-09T04:32:07.317090+0000","last_fullsized":"2023-10-09T04:32:07.317090+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.313987+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:32:07.313987+0000","last_peered":"2023-10-09T04:32:07.313987+0000","last_clean":"2023-10-09T04:32:07.313987+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:32:07.313987+0000","last_undegraded":"2023-10-09T04:32:07.313987+0000","last_fullsized":"2023-10-09T04:32:07.313987+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:10.535990+0000","last_change":"2023-10-09T04:29:12.641467+0000","last_active":"2023-10-09T04:32:10.535990+0000","last_peered":"2023-10-09T04:32:10.535990+0000","last_clean":"2023-10-09T04:32:10.535990+0000","last_became_active":"2023-10-09T04:29:12.641036+0000","last_became_peered":"2023-10-09T04:29:12.641036+0000","last_unstale":"2023-10-09T04:32:10.535990+0000","last_undegraded":"2023-10-09T04:32:10.535990+0000","last_fullsized":"2023-10-09T04:32:10.535990+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:41.624520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":89,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:10.542584+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:32:10.542584+0000","last_peered":"2023-10-09T04:32:10.542584+0000","last_clean":"2023-10-09T04:32:10.542584+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:32:10.542584+0000","last_undegraded":"2023-10-09T04:32:10.542584+0000","last_fullsized":"2023-10-09T04:32:10.542584+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.311363+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:32:07.311363+0000","last_peered":"2023-10-09T04:32:07.311363+0000","last_clean":"2023-10-09T04:32:07.311363+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:32:07.311363+0000","last_undegraded":"2023-10-09T04:32:07.311363+0000","last_fullsized":"2023-10-09T04:32:07.311363+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.039172+0000","last_change":"2023-10-09T04:28:27.618032+0000","last_active":"2023-10-09T04:32:07.039172+0000","last_peered":"2023-10-09T04:32:07.039172+0000","last_clean":"2023-10-09T04:28:19.577947+0000","last_became_active":"2023-10-09T04:28:27.618032+0000","last_became_peered":"2023-10-09T04:28:27.618032+0000","last_unstale":"2023-10-09T04:32:07.039172+0000","last_undegraded":"2023-10-09T04:32:07.039172+0000","last_fullsized":"2023-10-09T04:28:27.603354+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":94,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.312572+0000","last_change":"2023-10-09T04:28:27.599874+0000","last_active":"2023-10-09T04:32:07.312572+0000","last_peered":"2023-10-09T04:32:07.312572+0000","last_clean":"2023-10-09T04:28:19.571527+0000","last_became_active":"2023-10-09T04:28:27.599874+0000","last_became_peered":"2023-10-09T04:28:27.599874+0000","last_unstale":"2023-10-09T04:32:07.312572+0000","last_undegraded":"2023-10-09T04:32:07.312572+0000","last_fullsized":"2023-10-09T04:28:27.597863+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:45.767094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034367+0000","last_change":"2023-10-09T04:28:51.528372+0000","last_active":"2023-10-09T04:32:07.034367+0000","last_peered":"2023-10-09T04:32:07.034367+0000","last_clean":"2023-10-09T04:32:07.034367+0000","last_became_active":"2023-10-09T04:28:51.527988+0000","last_became_peered":"2023-10-09T04:28:51.527988+0000","last_unstale":"2023-10-09T04:32:07.034367+0000","last_undegraded":"2023-10-09T04:32:07.034367+0000","last_fullsized":"2023-10-09T04:32:07.034367+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.258888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":82,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.316605+0000","last_change":"2023-10-09T04:28:27.608355+0000","last_active":"2023-10-09T04:32:07.316605+0000","last_peered":"2023-10-09T04:32:07.316605+0000","last_clean":"2023-10-09T04:22:23.598572+0000","last_became_active":"2023-10-09T04:28:27.608355+0000","last_became_peered":"2023-10-09T04:28:27.608355+0000","last_unstale":"2023-10-09T04:32:07.316605+0000","last_undegraded":"2023-10-09T04:32:07.316605+0000","last_fullsized":"2023-10-09T04:28:27.604613+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.314942+0000","last_change":"2023-10-09T04:28:51.533081+0000","last_active":"2023-10-09T04:32:07.314942+0000","last_peered":"2023-10-09T04:32:07.314942+0000","last_clean":"2023-10-09T04:32:07.314942+0000","last_became_active":"2023-10-09T04:28:51.532684+0000","last_became_peered":"2023-10-09T04:28:51.532684+0000","last_unstale":"2023-10-09T04:32:07.314942+0000","last_undegraded":"2023-10-09T04:32:07.314942+0000","last_fullsized":"2023-10-09T04:32:07.314942+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316599+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:32:07.316599+0000","last_peered":"2023-10-09T04:32:07.316599+0000","last_clean":"2023-10-09T04:32:07.316599+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:32:07.316599+0000","last_undegraded":"2023-10-09T04:32:07.316599+0000","last_fullsized":"2023-10-09T04:32:07.316599+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:09.239142+0000","last_change":"2023-10-09T04:23:18.772738+0000","last_active":"2023-10-09T04:32:09.239142+0000","last_peered":"2023-10-09T04:32:09.239142+0000","last_clean":"2023-10-09T04:32:09.239142+0000","last_became_active":"2023-10-09T04:23:18.772392+0000","last_became_peered":"2023-10-09T04:23:18.772392+0000","last_unstale":"2023-10-09T04:32:09.239142+0000","last_undegraded":"2023-10-09T04:32:09.239142+0000","last_fullsized":"2023-10-09T04:32:09.239142+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:44.111777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":109,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317436+0000","last_change":"2023-10-09T04:11:26.215946+0000","last_active":"2023-10-09T04:32:07.317436+0000","last_peered":"2023-10-09T04:32:07.317436+0000","last_clean":"2023-10-09T04:32:07.317436+0000","last_became_active":"2023-10-09T04:11:26.215354+0000","last_became_peered":"2023-10-09T04:11:26.215354+0000","last_unstale":"2023-10-09T04:32:07.317436+0000","last_undegraded":"2023-10-09T04:32:07.317436+0000","last_fullsized":"2023-10-09T04:32:07.317436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317532+0000","last_change":"2023-10-09T04:22:50.700021+0000","last_active":"2023-10-09T04:32:07.317532+0000","last_peered":"2023-10-09T04:32:07.317532+0000","last_clean":"2023-10-09T04:32:07.317532+0000","last_became_active":"2023-10-09T04:22:50.699635+0000","last_became_peered":"2023-10-09T04:22:50.699635+0000","last_unstale":"2023-10-09T04:32:07.317532+0000","last_undegraded":"2023-10-09T04:32:07.317532+0000","last_fullsized":"2023-10-09T04:32:07.317532+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":104,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.315854+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:32:07.315854+0000","last_peered":"2023-10-09T04:32:07.315854+0000","last_clean":"2023-10-09T04:32:07.315854+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:32:07.315854+0000","last_undegraded":"2023-10-09T04:32:07.315854+0000","last_fullsized":"2023-10-09T04:32:07.315854+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034739+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:32:07.034739+0000","last_peered":"2023-10-09T04:32:07.034739+0000","last_clean":"2023-10-09T04:32:07.034739+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:32:07.034739+0000","last_undegraded":"2023-10-09T04:32:07.034739+0000","last_fullsized":"2023-10-09T04:32:07.034739+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":88,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312247+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:32:07.312247+0000","last_peered":"2023-10-09T04:32:07.312247+0000","last_clean":"2023-10-09T04:32:07.312247+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:32:07.312247+0000","last_undegraded":"2023-10-09T04:32:07.312247+0000","last_fullsized":"2023-10-09T04:32:07.312247+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.316959+0000","last_change":"2023-10-09T04:28:27.609484+0000","last_active":"2023-10-09T04:32:07.316959+0000","last_peered":"2023-10-09T04:32:07.316959+0000","last_clean":"2023-10-09T04:23:18.758860+0000","last_became_active":"2023-10-09T04:28:27.609484+0000","last_became_peered":"2023-10-09T04:28:27.609484+0000","last_unstale":"2023-10-09T04:32:07.316959+0000","last_undegraded":"2023-10-09T04:32:07.316959+0000","last_fullsized":"2023-10-09T04:28:27.606325+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.315187+0000","last_change":"2023-10-09T04:11:26.216393+0000","last_active":"2023-10-09T04:32:07.315187+0000","last_peered":"2023-10-09T04:32:07.315187+0000","last_clean":"2023-10-09T04:32:07.315187+0000","last_became_active":"2023-10-09T04:11:26.215918+0000","last_became_peered":"2023-10-09T04:11:26.215918+0000","last_unstale":"2023-10-09T04:32:07.315187+0000","last_undegraded":"2023-10-09T04:32:07.315187+0000","last_fullsized":"2023-10-09T04:32:07.315187+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":88,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312879+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:32:07.312879+0000","last_peered":"2023-10-09T04:32:07.312879+0000","last_clean":"2023-10-09T04:32:07.312879+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:32:07.312879+0000","last_undegraded":"2023-10-09T04:32:07.312879+0000","last_fullsized":"2023-10-09T04:32:07.312879+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":116,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.038496+0000","last_change":"2023-10-09T04:28:27.608819+0000","last_active":"2023-10-09T04:32:07.038496+0000","last_peered":"2023-10-09T04:32:07.038496+0000","last_clean":"2023-10-09T04:28:19.577481+0000","last_became_active":"2023-10-09T04:28:27.608819+0000","last_became_peered":"2023-10-09T04:28:27.608819+0000","last_unstale":"2023-10-09T04:32:07.038496+0000","last_undegraded":"2023-10-09T04:28:27.602582+0000","last_fullsized":"2023-10-09T04:28:27.602406+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":94,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316721+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:32:07.316721+0000","last_peered":"2023-10-09T04:32:07.316721+0000","last_clean":"2023-10-09T04:32:07.316721+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:32:07.316721+0000","last_undegraded":"2023-10-09T04:32:07.316721+0000","last_fullsized":"2023-10-09T04:32:07.316721+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"136'2573","reported_seq":2582,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:30.119907+0000","last_change":"2023-10-09T04:32:09.868297+0000","last_active":"2023-10-09T04:32:30.119907+0000","last_peered":"2023-10-09T04:32:30.119907+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.741268+0000","last_became_peered":"2023-10-09T04:31:52.741268+0000","last_unstale":"2023-10-09T04:32:30.119907+0000","last_undegraded":"2023-10-09T04:32:09.855630+0000","last_fullsized":"2023-10-09T04:31:52.738279+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2573,"log_dups_size":0,"ondisk_log_size":2573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624128,"num_objects":2573,"num_object_clones":0,"num_object_copies":7719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2573,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2573,"num_write_kb":164672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":2573}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317210+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:32:07.317210+0000","last_peered":"2023-10-09T04:32:07.317210+0000","last_clean":"2023-10-09T04:32:07.317210+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:32:07.317210+0000","last_undegraded":"2023-10-09T04:32:07.317210+0000","last_fullsized":"2023-10-09T04:32:07.317210+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":31,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.314746+0000","last_change":"2023-10-09T04:28:27.610146+0000","last_active":"2023-10-09T04:32:07.314746+0000","last_peered":"2023-10-09T04:32:07.314746+0000","last_clean":"2023-10-09T04:11:26.190073+0000","last_became_active":"2023-10-09T04:28:27.610146+0000","last_became_peered":"2023-10-09T04:28:27.610146+0000","last_unstale":"2023-10-09T04:32:07.314746+0000","last_undegraded":"2023-10-09T04:32:07.314746+0000","last_fullsized":"2023-10-09T04:28:27.604381+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"136'2479","reported_seq":2490,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.107617+0000","last_change":"2023-10-09T04:31:52.743124+0000","last_active":"2023-10-09T04:32:30.107617+0000","last_peered":"2023-10-09T04:32:30.107617+0000","last_clean":"2023-10-09T04:32:30.107617+0000","last_became_active":"2023-10-09T04:31:52.742658+0000","last_became_peered":"2023-10-09T04:31:52.742658+0000","last_unstale":"2023-10-09T04:32:30.107617+0000","last_undegraded":"2023-10-09T04:32:30.107617+0000","last_fullsized":"2023-10-09T04:32:30.107617+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2479,"log_dups_size":0,"ondisk_log_size":2479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2479,"num_write_kb":158656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316049+0000","last_change":"2023-10-09T04:29:12.722710+0000","last_active":"2023-10-09T04:32:07.316049+0000","last_peered":"2023-10-09T04:32:07.316049+0000","last_clean":"2023-10-09T04:32:07.316049+0000","last_became_active":"2023-10-09T04:29:12.635166+0000","last_became_peered":"2023-10-09T04:29:12.635166+0000","last_unstale":"2023-10-09T04:32:07.316049+0000","last_undegraded":"2023-10-09T04:32:07.316049+0000","last_fullsized":"2023-10-09T04:32:07.316049+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316476+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:32:07.316476+0000","last_peered":"2023-10-09T04:32:07.316476+0000","last_clean":"2023-10-09T04:32:07.316476+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:32:07.316476+0000","last_undegraded":"2023-10-09T04:32:07.316476+0000","last_fullsized":"2023-10-09T04:32:07.316476+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"136'2561","reported_seq":2573,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.091117+0000","last_change":"2023-10-09T04:31:52.742863+0000","last_active":"2023-10-09T04:32:30.091117+0000","last_peered":"2023-10-09T04:32:30.091117+0000","last_clean":"2023-10-09T04:32:30.091117+0000","last_became_active":"2023-10-09T04:31:52.742528+0000","last_became_peered":"2023-10-09T04:31:52.742528+0000","last_unstale":"2023-10-09T04:32:30.091117+0000","last_undegraded":"2023-10-09T04:32:30.091117+0000","last_fullsized":"2023-10-09T04:32:30.091117+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2561,"log_dups_size":0,"ondisk_log_size":2561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167837696,"num_objects":2561,"num_object_clones":0,"num_object_copies":7683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2561,"num_write_kb":163904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":94,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.315953+0000","last_change":"2023-10-09T04:28:27.610412+0000","last_active":"2023-10-09T04:32:07.315953+0000","last_peered":"2023-10-09T04:32:07.315953+0000","last_clean":"2023-10-09T04:28:19.573266+0000","last_became_active":"2023-10-09T04:28:27.610412+0000","last_became_peered":"2023-10-09T04:28:27.610412+0000","last_unstale":"2023-10-09T04:32:07.315953+0000","last_undegraded":"2023-10-09T04:28:27.605925+0000","last_fullsized":"2023-10-09T04:28:27.605804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.039530+0000","last_change":"2023-10-09T04:22:50.704599+0000","last_active":"2023-10-09T04:32:07.039530+0000","last_peered":"2023-10-09T04:32:07.039530+0000","last_clean":"2023-10-09T04:32:07.039530+0000","last_became_active":"2023-10-09T04:22:50.703970+0000","last_became_peered":"2023-10-09T04:22:50.703970+0000","last_unstale":"2023-10-09T04:32:07.039530+0000","last_undegraded":"2023-10-09T04:32:07.039530+0000","last_fullsized":"2023-10-09T04:32:07.039530+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"136'2879","reported_seq":2892,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:32.660575+0000","last_change":"2023-10-09T04:31:52.743807+0000","last_active":"2023-10-09T04:32:32.660575+0000","last_peered":"2023-10-09T04:32:32.660575+0000","last_clean":"2023-10-09T04:32:32.660575+0000","last_became_active":"2023-10-09T04:31:52.743450+0000","last_became_peered":"2023-10-09T04:31:52.743450+0000","last_unstale":"2023-10-09T04:32:32.660575+0000","last_undegraded":"2023-10-09T04:32:32.660575+0000","last_fullsized":"2023-10-09T04:32:32.660575+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2879,"log_dups_size":0,"ondisk_log_size":2879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:52.615776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188678144,"num_objects":2879,"num_object_clones":0,"num_object_copies":8637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2879,"num_write_kb":184256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":91,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.039297+0000","last_change":"2023-10-09T04:28:51.527885+0000","last_active":"2023-10-09T04:32:07.039297+0000","last_peered":"2023-10-09T04:32:07.039297+0000","last_clean":"2023-10-09T04:23:18.758933+0000","last_became_active":"2023-10-09T04:28:51.527885+0000","last_became_peered":"2023-10-09T04:28:51.527885+0000","last_unstale":"2023-10-09T04:32:07.039297+0000","last_undegraded":"2023-10-09T04:32:07.039297+0000","last_fullsized":"2023-10-09T04:28:51.525148+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312315+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:32:07.312315+0000","last_peered":"2023-10-09T04:32:07.312315+0000","last_clean":"2023-10-09T04:32:07.312315+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:32:07.312315+0000","last_undegraded":"2023-10-09T04:32:07.312315+0000","last_fullsized":"2023-10-09T04:32:07.312315+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"136'2530","reported_seq":2542,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.116972+0000","last_change":"2023-10-09T04:31:52.743032+0000","last_active":"2023-10-09T04:32:30.116972+0000","last_peered":"2023-10-09T04:32:30.116972+0000","last_clean":"2023-10-09T04:32:30.116972+0000","last_became_active":"2023-10-09T04:31:52.741647+0000","last_became_peered":"2023-10-09T04:31:52.741647+0000","last_unstale":"2023-10-09T04:32:30.116972+0000","last_undegraded":"2023-10-09T04:32:30.116972+0000","last_fullsized":"2023-10-09T04:32:30.116972+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2530,"log_dups_size":0,"ondisk_log_size":2530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2530,"num_write_kb":161920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":51,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312372+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:32:07.312372+0000","last_peered":"2023-10-09T04:32:07.312372+0000","last_clean":"2023-10-09T04:32:07.312372+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:32:07.312372+0000","last_undegraded":"2023-10-09T04:32:07.312372+0000","last_fullsized":"2023-10-09T04:32:07.312372+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317582+0000","last_change":"2023-10-09T04:24:00.827521+0000","last_active":"2023-10-09T04:32:07.317582+0000","last_peered":"2023-10-09T04:32:07.317582+0000","last_clean":"2023-10-09T04:32:07.317582+0000","last_became_active":"2023-10-09T04:24:00.826897+0000","last_became_peered":"2023-10-09T04:24:00.826897+0000","last_unstale":"2023-10-09T04:32:07.317582+0000","last_undegraded":"2023-10-09T04:32:07.317582+0000","last_fullsized":"2023-10-09T04:32:07.317582+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"136'2818","reported_seq":2827,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:32.811980+0000","last_change":"2023-10-09T04:31:52.739726+0000","last_active":"2023-10-09T04:32:32.811980+0000","last_peered":"2023-10-09T04:32:32.811980+0000","last_clean":"2023-10-09T04:32:32.811980+0000","last_became_active":"2023-10-09T04:31:52.739407+0000","last_became_peered":"2023-10-09T04:31:52.739407+0000","last_unstale":"2023-10-09T04:32:32.811980+0000","last_undegraded":"2023-10-09T04:32:32.811980+0000","last_fullsized":"2023-10-09T04:32:32.811980+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2818,"log_dups_size":0,"ondisk_log_size":2818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:47.453978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2818,"num_write_kb":180352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.039141+0000","last_change":"2023-10-09T04:22:50.704346+0000","last_active":"2023-10-09T04:32:07.039141+0000","last_peered":"2023-10-09T04:32:07.039141+0000","last_clean":"2023-10-09T04:32:07.039141+0000","last_became_active":"2023-10-09T04:22:50.703954+0000","last_became_peered":"2023-10-09T04:22:50.703954+0000","last_unstale":"2023-10-09T04:32:07.039141+0000","last_undegraded":"2023-10-09T04:32:07.039141+0000","last_fullsized":"2023-10-09T04:32:07.039141+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.313936+0000","last_change":"2023-10-09T04:29:12.637984+0000","last_active":"2023-10-09T04:32:07.313936+0000","last_peered":"2023-10-09T04:32:07.313936+0000","last_clean":"2023-10-09T04:32:07.313936+0000","last_became_active":"2023-10-09T04:29:12.637551+0000","last_became_peered":"2023-10-09T04:29:12.637551+0000","last_unstale":"2023-10-09T04:32:07.313936+0000","last_undegraded":"2023-10-09T04:32:07.313936+0000","last_fullsized":"2023-10-09T04:32:07.313936+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:19.948362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"136'2608","reported_seq":2617,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:30.129941+0000","last_change":"2023-10-09T04:32:09.862902+0000","last_active":"2023-10-09T04:32:30.129941+0000","last_peered":"2023-10-09T04:32:30.129941+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.739928+0000","last_became_peered":"2023-10-09T04:31:52.739928+0000","last_unstale":"2023-10-09T04:32:30.129941+0000","last_undegraded":"2023-10-09T04:32:09.856456+0000","last_fullsized":"2023-10-09T04:31:52.736855+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2608,"log_dups_size":0,"ondisk_log_size":2608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170917888,"num_objects":2608,"num_object_clones":0,"num_object_copies":7824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2608,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2608,"num_write_kb":166912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":2608}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316247+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:32:07.316247+0000","last_peered":"2023-10-09T04:32:07.316247+0000","last_clean":"2023-10-09T04:32:07.316247+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:32:07.316247+0000","last_undegraded":"2023-10-09T04:32:07.316247+0000","last_fullsized":"2023-10-09T04:32:07.316247+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317995+0000","last_change":"2023-10-09T04:22:50.700516+0000","last_active":"2023-10-09T04:32:07.317995+0000","last_peered":"2023-10-09T04:32:07.317995+0000","last_clean":"2023-10-09T04:32:07.317995+0000","last_became_active":"2023-10-09T04:22:50.699676+0000","last_became_peered":"2023-10-09T04:22:50.699676+0000","last_unstale":"2023-10-09T04:32:07.317995+0000","last_undegraded":"2023-10-09T04:32:07.317995+0000","last_fullsized":"2023-10-09T04:32:07.317995+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"136'2434","reported_seq":2443,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:29.241375+0000","last_change":"2023-10-09T04:32:09.879049+0000","last_active":"2023-10-09T04:32:29.241375+0000","last_peered":"2023-10-09T04:32:29.241375+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.733735+0000","last_became_peered":"2023-10-09T04:31:52.733735+0000","last_unstale":"2023-10-09T04:32:29.241375+0000","last_undegraded":"2023-10-09T04:32:09.869011+0000","last_fullsized":"2023-10-09T04:31:52.730251+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2434,"log_dups_size":0,"ondisk_log_size":2434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:34.729548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2434,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2434,"num_write_kb":155776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":2434}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":96,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034049+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:32:07.034049+0000","last_peered":"2023-10-09T04:32:07.034049+0000","last_clean":"2023-10-09T04:32:07.034049+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:32:07.034049+0000","last_undegraded":"2023-10-09T04:32:07.034049+0000","last_fullsized":"2023-10-09T04:32:07.034049+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.035091+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:32:07.035091+0000","last_peered":"2023-10-09T04:32:07.035091+0000","last_clean":"2023-10-09T04:32:07.035091+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:32:07.035091+0000","last_undegraded":"2023-10-09T04:32:07.035091+0000","last_fullsized":"2023-10-09T04:32:07.035091+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"136'2888","reported_seq":2898,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:32.656505+0000","last_change":"2023-10-09T04:31:52.736550+0000","last_active":"2023-10-09T04:32:32.656505+0000","last_peered":"2023-10-09T04:32:32.656505+0000","last_clean":"2023-10-09T04:32:32.656505+0000","last_became_active":"2023-10-09T04:31:52.736098+0000","last_became_peered":"2023-10-09T04:31:52.736098+0000","last_unstale":"2023-10-09T04:32:32.656505+0000","last_undegraded":"2023-10-09T04:32:32.656505+0000","last_fullsized":"2023-10-09T04:32:32.656505+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2888,"log_dups_size":0,"ondisk_log_size":2888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:41.783315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189267968,"num_objects":2888,"num_object_clones":0,"num_object_copies":8664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2888,"num_write_kb":184832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":508,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.035101+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:32:07.035101+0000","last_peered":"2023-10-09T04:32:07.035101+0000","last_clean":"2023-10-09T04:32:07.035101+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:32:07.035101+0000","last_undegraded":"2023-10-09T04:32:07.035101+0000","last_fullsized":"2023-10-09T04:32:07.035101+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.313815+0000","last_change":"2023-10-09T04:28:51.527252+0000","last_active":"2023-10-09T04:32:07.313815+0000","last_peered":"2023-10-09T04:32:07.313815+0000","last_clean":"2023-10-09T04:23:18.768463+0000","last_became_active":"2023-10-09T04:28:51.527252+0000","last_became_peered":"2023-10-09T04:28:51.527252+0000","last_unstale":"2023-10-09T04:32:07.313815+0000","last_undegraded":"2023-10-09T04:32:07.313815+0000","last_fullsized":"2023-10-09T04:28:51.524832+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:29.101177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.035963+0000","last_change":"2023-10-09T04:28:27.600497+0000","last_active":"2023-10-09T04:32:07.035963+0000","last_peered":"2023-10-09T04:32:07.035963+0000","last_clean":"2023-10-09T04:28:19.561863+0000","last_became_active":"2023-10-09T04:28:27.600497+0000","last_became_peered":"2023-10-09T04:28:27.600497+0000","last_unstale":"2023-10-09T04:32:07.035963+0000","last_undegraded":"2023-10-09T04:32:07.035963+0000","last_fullsized":"2023-10-09T04:28:27.591503+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:26.054422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"136'2478","reported_seq":2490,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.115943+0000","last_change":"2023-10-09T04:31:52.744989+0000","last_active":"2023-10-09T04:32:30.115943+0000","last_peered":"2023-10-09T04:32:30.115943+0000","last_clean":"2023-10-09T04:32:30.115943+0000","last_became_active":"2023-10-09T04:31:52.744622+0000","last_became_peered":"2023-10-09T04:31:52.744622+0000","last_unstale":"2023-10-09T04:32:30.115943+0000","last_undegraded":"2023-10-09T04:32:30.115943+0000","last_fullsized":"2023-10-09T04:32:30.115943+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":596,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.034204+0000","last_change":"2023-10-09T04:28:27.605360+0000","last_active":"2023-10-09T04:32:07.034204+0000","last_peered":"2023-10-09T04:32:07.034204+0000","last_clean":"2023-10-09T04:13:52.717465+0000","last_became_active":"2023-10-09T04:28:27.605360+0000","last_became_peered":"2023-10-09T04:28:27.605360+0000","last_unstale":"2023-10-09T04:32:07.034204+0000","last_undegraded":"2023-10-09T04:28:27.591227+0000","last_fullsized":"2023-10-09T04:28:27.591053+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:40.508309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.317587+0000","last_change":"2023-10-09T04:22:23.610902+0000","last_active":"2023-10-09T04:32:07.317587+0000","last_peered":"2023-10-09T04:32:07.317587+0000","last_clean":"2023-10-09T04:32:07.317587+0000","last_became_active":"2023-10-09T04:22:23.610475+0000","last_became_peered":"2023-10-09T04:22:23.610475+0000","last_unstale":"2023-10-09T04:32:07.317587+0000","last_undegraded":"2023-10-09T04:32:07.317587+0000","last_fullsized":"2023-10-09T04:32:07.317587+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":98,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034835+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:32:07.034835+0000","last_peered":"2023-10-09T04:32:07.034835+0000","last_clean":"2023-10-09T04:32:07.034835+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:32:07.034835+0000","last_undegraded":"2023-10-09T04:32:07.034835+0000","last_fullsized":"2023-10-09T04:32:07.034835+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"136'2517","reported_seq":2529,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.108294+0000","last_change":"2023-10-09T04:31:52.744941+0000","last_active":"2023-10-09T04:32:30.108294+0000","last_peered":"2023-10-09T04:32:30.108294+0000","last_clean":"2023-10-09T04:32:30.108294+0000","last_became_active":"2023-10-09T04:31:52.744570+0000","last_became_peered":"2023-10-09T04:31:52.744570+0000","last_unstale":"2023-10-09T04:32:30.108294+0000","last_undegraded":"2023-10-09T04:32:30.108294+0000","last_fullsized":"2023-10-09T04:32:30.108294+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2517,"log_dups_size":0,"ondisk_log_size":2517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2517,"num_write_kb":161088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1244,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312417+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:32:07.312417+0000","last_peered":"2023-10-09T04:32:07.312417+0000","last_clean":"2023-10-09T04:32:07.312417+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:32:07.312417+0000","last_undegraded":"2023-10-09T04:32:07.312417+0000","last_fullsized":"2023-10-09T04:32:07.312417+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034207+0000","last_change":"2023-10-09T04:23:18.768540+0000","last_active":"2023-10-09T04:32:07.034207+0000","last_peered":"2023-10-09T04:32:07.034207+0000","last_clean":"2023-10-09T04:32:07.034207+0000","last_became_active":"2023-10-09T04:23:18.768086+0000","last_became_peered":"2023-10-09T04:23:18.768086+0000","last_unstale":"2023-10-09T04:32:07.034207+0000","last_undegraded":"2023-10-09T04:32:07.034207+0000","last_fullsized":"2023-10-09T04:32:07.034207+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:30.410448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":119,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.034877+0000","last_change":"2023-10-09T04:28:27.599081+0000","last_active":"2023-10-09T04:32:07.034877+0000","last_peered":"2023-10-09T04:32:07.034877+0000","last_clean":"2023-10-09T04:28:19.562734+0000","last_became_active":"2023-10-09T04:28:27.599081+0000","last_became_peered":"2023-10-09T04:28:27.599081+0000","last_unstale":"2023-10-09T04:32:07.034877+0000","last_undegraded":"2023-10-09T04:32:07.034877+0000","last_fullsized":"2023-10-09T04:28:27.592506+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:57.226039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"136'2617","reported_seq":2625,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:30.111799+0000","last_change":"2023-10-09T04:32:09.866092+0000","last_active":"2023-10-09T04:32:30.111799+0000","last_peered":"2023-10-09T04:32:30.111799+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.740185+0000","last_became_peered":"2023-10-09T04:31:52.740185+0000","last_unstale":"2023-10-09T04:32:30.111799+0000","last_undegraded":"2023-10-09T04:32:09.859653+0000","last_fullsized":"2023-10-09T04:31:52.735459+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2617,"log_dups_size":0,"ondisk_log_size":2617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2617,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":2617}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":572,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.312934+0000","last_change":"2023-10-09T04:28:27.614054+0000","last_active":"2023-10-09T04:32:07.312934+0000","last_peered":"2023-10-09T04:32:07.312934+0000","last_clean":"2023-10-09T04:13:52.699710+0000","last_became_active":"2023-10-09T04:28:27.614054+0000","last_became_peered":"2023-10-09T04:28:27.614054+0000","last_unstale":"2023-10-09T04:32:07.312934+0000","last_undegraded":"2023-10-09T04:28:27.594108+0000","last_fullsized":"2023-10-09T04:28:27.593983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:21.875430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":338,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.315273+0000","last_change":"2023-10-09T04:22:23.612486+0000","last_active":"2023-10-09T04:32:07.315273+0000","last_peered":"2023-10-09T04:32:07.315273+0000","last_clean":"2023-10-09T04:32:07.315273+0000","last_became_active":"2023-10-09T04:22:23.611991+0000","last_became_peered":"2023-10-09T04:22:23.611991+0000","last_unstale":"2023-10-09T04:32:07.315273+0000","last_undegraded":"2023-10-09T04:32:07.315273+0000","last_fullsized":"2023-10-09T04:32:07.315273+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.039697+0000","last_change":"2023-10-09T04:28:51.527731+0000","last_active":"2023-10-09T04:32:07.039697+0000","last_peered":"2023-10-09T04:32:07.039697+0000","last_clean":"2023-10-09T04:28:19.566551+0000","last_became_active":"2023-10-09T04:28:51.527731+0000","last_became_peered":"2023-10-09T04:28:51.527731+0000","last_unstale":"2023-10-09T04:32:07.039697+0000","last_undegraded":"2023-10-09T04:32:07.039697+0000","last_fullsized":"2023-10-09T04:28:51.525113+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.039315+0000","last_change":"2023-10-09T04:11:27.723673+0000","last_active":"2023-10-09T04:32:07.039315+0000","last_peered":"2023-10-09T04:32:07.039315+0000","last_clean":"2023-10-09T04:32:07.039315+0000","last_became_active":"2023-10-09T04:11:27.723262+0000","last_became_peered":"2023-10-09T04:11:27.723262+0000","last_unstale":"2023-10-09T04:32:07.039315+0000","last_undegraded":"2023-10-09T04:32:07.039315+0000","last_fullsized":"2023-10-09T04:32:07.039315+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"136'2498","reported_seq":2508,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:29.224520+0000","last_change":"2023-10-09T04:32:09.866362+0000","last_active":"2023-10-09T04:32:29.224520+0000","last_peered":"2023-10-09T04:32:29.224520+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.731757+0000","last_became_peered":"2023-10-09T04:31:52.731757+0000","last_unstale":"2023-10-09T04:32:29.224520+0000","last_undegraded":"2023-10-09T04:32:09.854852+0000","last_fullsized":"2023-10-09T04:31:52.728894+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2498,"log_dups_size":0,"ondisk_log_size":2498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:57.001512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163708928,"num_objects":2498,"num_object_clones":0,"num_object_copies":7494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2498,"num_write_kb":159872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":2498}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":680,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.315541+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:32:07.315541+0000","last_peered":"2023-10-09T04:32:07.315541+0000","last_clean":"2023-10-09T04:32:07.315541+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:32:07.315541+0000","last_undegraded":"2023-10-09T04:32:07.315541+0000","last_fullsized":"2023-10-09T04:32:07.315541+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":98,"reported_epoch":135,"state":"active+clean+remapped","last_fresh":"2023-10-09T04:32:07.311716+0000","last_change":"2023-10-09T04:29:13.767749+0000","last_active":"2023-10-09T04:32:07.311716+0000","last_peered":"2023-10-09T04:32:07.311716+0000","last_clean":"2023-10-09T04:32:07.311716+0000","last_became_active":"2023-10-09T04:29:13.767307+0000","last_became_peered":"2023-10-09T04:29:13.767307+0000","last_unstale":"2023-10-09T04:32:07.311716+0000","last_undegraded":"2023-10-09T04:32:07.311716+0000","last_fullsized":"2023-10-09T04:32:07.311716+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:30.355543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.313189+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:32:07.313189+0000","last_peered":"2023-10-09T04:32:07.313189+0000","last_clean":"2023-10-09T04:32:07.313189+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:32:07.313189+0000","last_undegraded":"2023-10-09T04:32:07.313189+0000","last_fullsized":"2023-10-09T04:32:07.313189+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"136'2476","reported_seq":2489,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:29.231481+0000","last_change":"2023-10-09T04:31:52.742319+0000","last_active":"2023-10-09T04:32:29.231481+0000","last_peered":"2023-10-09T04:32:29.231481+0000","last_clean":"2023-10-09T04:32:29.231481+0000","last_became_active":"2023-10-09T04:31:52.742043+0000","last_became_peered":"2023-10-09T04:31:52.742043+0000","last_unstale":"2023-10-09T04:32:29.231481+0000","last_undegraded":"2023-10-09T04:32:29.231481+0000","last_fullsized":"2023-10-09T04:32:29.231481+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2476,"log_dups_size":0,"ondisk_log_size":2476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:18.784653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162267136,"num_objects":2476,"num_object_clones":0,"num_object_copies":7428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2476,"num_write_kb":158464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":349,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316359+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:32:07.316359+0000","last_peered":"2023-10-09T04:32:07.316359+0000","last_clean":"2023-10-09T04:32:07.316359+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:32:07.316359+0000","last_undegraded":"2023-10-09T04:32:07.316359+0000","last_fullsized":"2023-10-09T04:32:07.316359+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.034957+0000","last_change":"2023-10-09T04:28:27.599163+0000","last_active":"2023-10-09T04:32:07.034957+0000","last_peered":"2023-10-09T04:32:07.034957+0000","last_clean":"2023-10-09T04:28:19.563114+0000","last_became_active":"2023-10-09T04:28:27.599163+0000","last_became_peered":"2023-10-09T04:28:27.599163+0000","last_unstale":"2023-10-09T04:32:07.034957+0000","last_undegraded":"2023-10-09T04:32:07.034957+0000","last_fullsized":"2023-10-09T04:28:27.591374+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:37.443801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":79,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.312512+0000","last_change":"2023-10-09T04:28:27.610457+0000","last_active":"2023-10-09T04:32:07.312512+0000","last_peered":"2023-10-09T04:32:07.312512+0000","last_clean":"2023-10-09T04:28:19.571509+0000","last_became_active":"2023-10-09T04:28:27.610457+0000","last_became_peered":"2023-10-09T04:28:27.610457+0000","last_unstale":"2023-10-09T04:32:07.312512+0000","last_undegraded":"2023-10-09T04:32:07.312512+0000","last_fullsized":"2023-10-09T04:28:27.598190+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:23.684166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"136'2872","reported_seq":2885,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:32.809548+0000","last_change":"2023-10-09T04:31:52.742914+0000","last_active":"2023-10-09T04:32:32.809548+0000","last_peered":"2023-10-09T04:32:32.809548+0000","last_clean":"2023-10-09T04:32:32.809548+0000","last_became_active":"2023-10-09T04:31:52.742616+0000","last_became_peered":"2023-10-09T04:31:52.742616+0000","last_unstale":"2023-10-09T04:32:32.809548+0000","last_undegraded":"2023-10-09T04:32:32.809548+0000","last_fullsized":"2023-10-09T04:32:32.809548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2872,"log_dups_size":0,"ondisk_log_size":2872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:07.705518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188219392,"num_objects":2872,"num_object_clones":0,"num_object_copies":8616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2872,"num_write_kb":183808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":50,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316508+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:32:07.316508+0000","last_peered":"2023-10-09T04:32:07.316508+0000","last_clean":"2023-10-09T04:32:07.316508+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:32:07.316508+0000","last_undegraded":"2023-10-09T04:32:07.316508+0000","last_fullsized":"2023-10-09T04:32:07.316508+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":359,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.035506+0000","last_change":"2023-10-09T04:22:50.703526+0000","last_active":"2023-10-09T04:32:07.035506+0000","last_peered":"2023-10-09T04:32:07.035506+0000","last_clean":"2023-10-09T04:32:07.035506+0000","last_became_active":"2023-10-09T04:22:50.700931+0000","last_became_peered":"2023-10-09T04:22:50.700931+0000","last_unstale":"2023-10-09T04:32:07.035506+0000","last_undegraded":"2023-10-09T04:32:07.035506+0000","last_fullsized":"2023-10-09T04:32:07.035506+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:15.957299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.315005+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:32:07.315005+0000","last_peered":"2023-10-09T04:32:07.315005+0000","last_clean":"2023-10-09T04:32:07.315005+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:32:07.315005+0000","last_undegraded":"2023-10-09T04:32:07.315005+0000","last_fullsized":"2023-10-09T04:32:07.315005+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.3","version":"136'2519","reported_seq":2531,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:30.123730+0000","last_change":"2023-10-09T04:31:52.746000+0000","last_active":"2023-10-09T04:32:30.123730+0000","last_peered":"2023-10-09T04:32:30.123730+0000","last_clean":"2023-10-09T04:32:30.123730+0000","last_became_active":"2023-10-09T04:31:52.745674+0000","last_became_peered":"2023-10-09T04:31:52.745674+0000","last_unstale":"2023-10-09T04:32:30.123730+0000","last_undegraded":"2023-10-09T04:32:30.123730+0000","last_fullsized":"2023-10-09T04:32:30.123730+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":2519,"log_dups_size":0,"ondisk_log_size":2519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165085184,"num_objects":2519,"num_object_clones":0,"num_object_copies":7557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2519,"num_write_kb":161216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":183,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.039122+0000","last_change":"2023-10-09T04:23:18.770289+0000","last_active":"2023-10-09T04:32:07.039122+0000","last_peered":"2023-10-09T04:32:07.039122+0000","last_clean":"2023-10-09T04:32:07.039122+0000","last_became_active":"2023-10-09T04:23:18.769804+0000","last_became_peered":"2023-10-09T04:23:18.769804+0000","last_unstale":"2023-10-09T04:32:07.039122+0000","last_undegraded":"2023-10-09T04:32:07.039122+0000","last_fullsized":"2023-10-09T04:32:07.039122+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":63,"num_read_kb":1369,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.313043+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:32:07.313043+0000","last_peered":"2023-10-09T04:32:07.313043+0000","last_clean":"2023-10-09T04:32:07.313043+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:32:07.313043+0000","last_undegraded":"2023-10-09T04:32:07.313043+0000","last_fullsized":"2023-10-09T04:32:07.313043+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.316267+0000","last_change":"2023-10-09T04:28:27.605015+0000","last_active":"2023-10-09T04:32:07.316267+0000","last_peered":"2023-10-09T04:32:07.316267+0000","last_clean":"2023-10-09T04:28:19.571550+0000","last_became_active":"2023-10-09T04:28:27.605015+0000","last_became_peered":"2023-10-09T04:28:27.605015+0000","last_unstale":"2023-10-09T04:32:07.316267+0000","last_undegraded":"2023-10-09T04:32:07.316267+0000","last_fullsized":"2023-10-09T04:28:27.598518+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":77,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T04:32:10.541754+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:32:10.541754+0000","last_peered":"2023-10-09T04:32:10.541754+0000","last_clean":"2023-10-09T04:32:10.541754+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:32:10.541754+0000","last_undegraded":"2023-10-09T04:32:10.541754+0000","last_fullsized":"2023-10-09T04:32:10.541754+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":96,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-10-09T04:32:07.312068+0000","last_change":"2023-10-09T04:28:27.600435+0000","last_active":"2023-10-09T04:32:07.312068+0000","last_peered":"2023-10-09T04:32:07.312068+0000","last_clean":"2023-10-09T04:28:19.572818+0000","last_became_active":"2023-10-09T04:28:27.600435+0000","last_became_peered":"2023-10-09T04:28:27.600435+0000","last_unstale":"2023-10-09T04:32:07.312068+0000","last_undegraded":"2023-10-09T04:32:07.312068+0000","last_fullsized":"2023-10-09T04:28:27.592390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:45.015770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":41,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.317421+0000","last_change":"2023-10-09T04:28:27.600816+0000","last_active":"2023-10-09T04:32:07.317421+0000","last_peered":"2023-10-09T04:32:07.317421+0000","last_clean":"2023-10-09T04:11:28.922252+0000","last_became_active":"2023-10-09T04:28:27.600816+0000","last_became_peered":"2023-10-09T04:28:27.600816+0000","last_unstale":"2023-10-09T04:32:07.317421+0000","last_undegraded":"2023-10-09T04:28:27.597550+0000","last_fullsized":"2023-10-09T04:28:27.597399+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.034854+0000","last_change":"2023-10-09T04:29:12.632856+0000","last_active":"2023-10-09T04:32:07.034854+0000","last_peered":"2023-10-09T04:32:07.034854+0000","last_clean":"2023-10-09T04:32:07.034854+0000","last_became_active":"2023-10-09T04:29:12.632465+0000","last_became_peered":"2023-10-09T04:29:12.632465+0000","last_unstale":"2023-10-09T04:32:07.034854+0000","last_undegraded":"2023-10-09T04:32:07.034854+0000","last_fullsized":"2023-10-09T04:32:07.034854+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:26.854929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":41,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.318005+0000","last_change":"2023-10-09T04:28:27.606662+0000","last_active":"2023-10-09T04:32:07.318005+0000","last_peered":"2023-10-09T04:32:07.318005+0000","last_clean":"2023-10-09T04:11:28.923128+0000","last_became_active":"2023-10-09T04:28:27.606662+0000","last_became_peered":"2023-10-09T04:28:27.606662+0000","last_unstale":"2023-10-09T04:32:07.318005+0000","last_undegraded":"2023-10-09T04:28:27.599069+0000","last_fullsized":"2023-10-09T04:28:27.598215+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312187+0000","last_change":"2023-10-09T04:29:12.632656+0000","last_active":"2023-10-09T04:32:07.312187+0000","last_peered":"2023-10-09T04:32:07.312187+0000","last_clean":"2023-10-09T04:32:07.312187+0000","last_became_active":"2023-10-09T04:29:12.632273+0000","last_became_peered":"2023-10-09T04:29:12.632273+0000","last_unstale":"2023-10-09T04:32:07.312187+0000","last_undegraded":"2023-10-09T04:32:07.312187+0000","last_fullsized":"2023-10-09T04:32:07.312187+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:27.569523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":39,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.038921+0000","last_change":"2023-10-09T04:28:51.529660+0000","last_active":"2023-10-09T04:32:07.038921+0000","last_peered":"2023-10-09T04:32:07.038921+0000","last_clean":"2023-10-09T04:32:07.038921+0000","last_became_active":"2023-10-09T04:28:51.529256+0000","last_became_peered":"2023-10-09T04:28:51.529256+0000","last_unstale":"2023-10-09T04:32:07.038921+0000","last_undegraded":"2023-10-09T04:32:07.038921+0000","last_fullsized":"2023-10-09T04:32:07.038921+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.312218+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:32:07.312218+0000","last_peered":"2023-10-09T04:32:07.312218+0000","last_clean":"2023-10-09T04:32:07.312218+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:32:07.312218+0000","last_undegraded":"2023-10-09T04:32:07.312218+0000","last_fullsized":"2023-10-09T04:32:07.312218+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":98,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:32:07.312120+0000","last_change":"2023-10-09T04:28:27.606014+0000","last_active":"2023-10-09T04:32:07.312120+0000","last_peered":"2023-10-09T04:32:07.312120+0000","last_clean":"2023-10-09T04:28:19.571450+0000","last_became_active":"2023-10-09T04:28:27.606014+0000","last_became_peered":"2023-10-09T04:28:27.606014+0000","last_unstale":"2023-10-09T04:32:07.312120+0000","last_undegraded":"2023-10-09T04:28:27.597046+0000","last_fullsized":"2023-10-09T04:28:27.594857+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:07.466276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:32:07.316138+0000","last_change":"2023-10-09T04:22:50.700234+0000","last_active":"2023-10-09T04:32:07.316138+0000","last_peered":"2023-10-09T04:32:07.316138+0000","last_clean":"2023-10-09T04:32:07.316138+0000","last_became_active":"2023-10-09T04:22:50.699823+0000","last_became_peered":"2023-10-09T04:22:50.699823+0000","last_unstale":"2023-10-09T04:32:07.316138+0000","last_undegraded":"2023-10-09T04:32:07.316138+0000","last_fullsized":"2023-10-09T04:32:07.316138+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2735931392,"num_objects":41747,"num_object_clones":0,"num_object_copies":125241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41747,"num_write_kb":2671808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7316439040,"data_stored":7316439040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41747,"ondisk_log_size":41747,"up":43,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13247,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430497,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1268532,"kb_used_data":954924,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92476620,"statfs":{"total":95995035648,"available":94696058880,"internally_reserved":0,"allocated":977842176,"data_stored":977317589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:31:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":3.6240000000000001,"15min":1.802},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":16.806999999999999,"5min":250.65700000000001,"15min":250.65700000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.2570000000000001,"5min":4.3250000000000002,"15min":2.0619999999999998},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.952000000000002,"5min":251.214,"15min":251.214},"last":9.8529999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:32:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1100000000000001,"15min":0.98799999999999999},"min":{"1min":0.42499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1679999999999999,"5min":7.282,"15min":7.282},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.371,"5min":1.5900000000000001,"15min":1.159},"min":{"1min":0.33200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.988,"5min":11.988,"15min":11.988},"last":1.7809999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:31:43 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":2.5289999999999999,"15min":1.46},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.151,"5min":106.456,"15min":106.456},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.601,"5min":2.6850000000000001,"15min":1.552},"min":{"1min":0.44500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":12.177,"5min":106.708,"15min":106.708},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:32:11 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.1160000000000001,"15min":1.054},"min":{"1min":0.501,"5min":0.318,"15min":0.318},"max":{"1min":1.423,"5min":5.3390000000000004,"15min":5.3390000000000004},"last":1.5529999999999999},{"interface":"front","average":{"1min":0.83399999999999996,"5min":1.115,"15min":1.071},"min":{"1min":0.46300000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.2310000000000001,"5min":7.3399999999999999,"15min":7.3399999999999999},"last":1.728}]},{"osd":4,"last update":"Mon Oct 9 04:32:18 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.5209999999999999,"15min":1.121},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.3640000000000001,"5min":6.5780000000000003,"15min":6.5780000000000003},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.163,"15min":1.04},"min":{"1min":0.40300000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.758,"5min":5.431,"15min":5.431},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.0600000000000001,"15min":1.0089999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.0209999999999999,"5min":4.0209999999999999,"15min":4.0209999999999999},"last":5.4470000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.2,"15min":1.0800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":1.9219999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:32:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.3959999999999999,"15min":1.1759999999999999},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2170000000000001,"5min":6.5460000000000003,"15min":6.5460000000000003},"last":1.994},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1020000000000001,"15min":1.0720000000000001},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.601,"5min":5.4050000000000002,"15min":5.4050000000000002},"last":0.57099999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626730,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289516,"kb_used_data":980004,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92455636,"statfs":{"total":95995035648,"available":94674571264,"internally_reserved":0,"allocated":1003524096,"data_stored":1003007574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":3.6749999999999998,"15min":1.8200000000000001},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":4.7629999999999999,"5min":147.25800000000001,"15min":147.25800000000001},"last":1.8560000000000001},{"interface":"front","average":{"1min":1.8859999999999999,"5min":3.7040000000000002,"15min":1.8140000000000001},"min":{"1min":0.32200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":9.4570000000000007,"5min":147.37,"15min":147.37},"last":0.90300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:32:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.268,"15min":1.016},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.851,"5min":8.2690000000000001,"15min":8.2690000000000001},"last":0.624},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.2989999999999999,"15min":1.036},"min":{"1min":0.32900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.5249999999999999,"5min":10.099,"15min":10.099},"last":1.0069999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":4.1550000000000002,"15min":2.0270000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.603999999999999,"5min":141.93899999999999,"15min":141.93899999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.8420000000000001,"5min":4.0190000000000001,"15min":1.9570000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.606999999999999,"5min":141.898,"15min":141.898},"last":1.0940000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":1.2729999999999999,"15min":1.036},"min":{"1min":0.35899999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1950000000000001,"5min":10.029,"15min":10.029},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.2010000000000001,"15min":1.0940000000000001},"min":{"1min":0.47399999999999998,"5min":0.378,"15min":0.378},"max":{"1min":4.6820000000000004,"5min":6.0519999999999996,"15min":6.0519999999999996},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.2749999999999999,"15min":1.0329999999999999},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.315},"max":{"1min":4.7380000000000004,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.139,"15min":1.0269999999999999},"min":{"1min":0.371,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.7510000000000003,"5min":11.018000000000001,"15min":11.018000000000001},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.0509999999999999,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.7120000000000002,"5min":3.7120000000000002,"15min":3.9180000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.0670000000000002,"5min":1.242,"15min":1.0780000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.25,"15min":1.1259999999999999},"min":{"1min":0.38,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.2030000000000003,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.3,"5min":1.3200000000000001,"15min":1.212},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.726,"5min":8.9139999999999997,"15min":8.9139999999999997},"last":0.82199999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958646,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":885556,"kb_used_data":755436,"kb_used_omap":6,"kb_used_meta":130105,"kb_avail":92859596,"statfs":{"total":95995035648,"available":95088226304,"internally_reserved":0,"allocated":773566464,"data_stored":772926433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":133227532},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":3.6909999999999998,"15min":1.6830000000000001},"min":{"1min":0.23100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.524,"5min":158.435,"15min":158.435},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.484,"5min":4.0330000000000004,"15min":1.8320000000000001},"min":{"1min":0.34200000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.9009999999999998,"5min":159.38999999999999,"15min":159.38999999999999},"last":0.72199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:31:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":3.633,"15min":1.6559999999999999},"min":{"1min":0.30599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.282,"5min":157.70400000000001,"15min":157.70400000000001},"last":3.5329999999999999},{"interface":"front","average":{"1min":2.782,"5min":3.7229999999999999,"15min":1.764},"min":{"1min":0.41799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.632999999999999,"5min":157.64099999999999,"15min":157.64099999999999},"last":0.376}]},{"osd":3,"last update":"Mon Oct 9 04:32:20 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.5149999999999999,"15min":1.0569999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5860000000000003,"5min":7.9960000000000004,"15min":7.9960000000000004},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.448,"15min":0.93899999999999995},"min":{"1min":0.34499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":5.7229999999999999,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.69199999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.5149999999999999,"15min":1.123},"min":{"1min":0.38800000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.875,"5min":10.372999999999999,"15min":10.372999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.6819999999999999,"15min":1.25},"min":{"1min":0.60299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.2480000000000002,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.9449999999999998,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.99099999999999999,"15min":0.98899999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7069999999999999,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.4810000000000001,"15min":1.177},"min":{"1min":0.41299999999999998,"5min":0.374,"15min":0.374},"max":{"1min":3.8239999999999998,"5min":9.6890000000000001,"15min":9.6890000000000001},"last":0.745},{"interface":"front","average":{"1min":0.751,"5min":1.3220000000000001,"15min":1.121},"min":{"1min":0.41799999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0209999999999999,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.4419999999999999,"15min":1.153},"min":{"1min":0.442,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0189999999999999,"5min":6.3399999999999999,"15min":6.3399999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.26,"5min":1.591,"15min":1.2709999999999999},"min":{"1min":0.59899999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.266,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.59499999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122172,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913656,"kb_used_data":729712,"kb_used_omap":9,"kb_used_meta":183926,"kb_avail":92831496,"statfs":{"total":95995035648,"available":95059451904,"internally_reserved":0,"allocated":747225088,"data_stored":746582818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":188340721},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:32:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.577,"15min":0.95399999999999996},"min":{"1min":0.32000000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":1.171,"5min":43.216000000000001,"15min":43.216000000000001},"last":2.9950000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.839,"15min":1.052},"min":{"1min":0.375,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.1789999999999998,"5min":43.091999999999999,"15min":43.091999999999999},"last":1.018}]},{"osd":2,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":1.9490000000000001,"15min":1.0900000000000001},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.1379999999999999,"5min":37.453000000000003,"15min":37.453000000000003},"last":2.9340000000000002},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.5600000000000001,"15min":1.04},"min":{"1min":0.432,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.1689999999999996,"5min":25.178999999999998,"15min":25.178999999999998},"last":0.32900000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:32:17 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.93899999999999995,"15min":0.86199999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.075,"5min":7.5419999999999998,"15min":7.5419999999999998},"last":3.0289999999999999},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.3140000000000001,"15min":0.91100000000000003},"min":{"1min":0.47599999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.7330000000000001,"5min":7.5250000000000004,"15min":7.5250000000000004},"last":2.7490000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.7529999999999999,"15min":1.278},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.2039999999999997,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":0.624},{"interface":"front","average":{"1min":0.879,"5min":1.196,"15min":1.0920000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.4409999999999998,"5min":7.5880000000000001,"15min":7.5880000000000001},"last":0.42499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.034,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":3.6339999999999999,"5min":3.6339999999999999,"15min":3.6339999999999999},"last":0.441},{"interface":"front","average":{"1min":1.238,"5min":1.0369999999999999,"15min":1.038},"min":{"1min":0.65500000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":3.343,"5min":3.343,"15min":3.343},"last":2.0150000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.4399999999999999,"15min":1.1619999999999999},"min":{"1min":0.47099999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.3659999999999997,"5min":7.508,"15min":7.508},"last":1.3939999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.369,"15min":1.1439999999999999},"min":{"1min":0.442,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.4129999999999998,"5min":23.994,"15min":23.994},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":1.9510000000000001,"15min":1.3939999999999999},"min":{"1min":0.443,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.9600000000000009,"5min":10.528,"15min":10.528},"last":1.4399999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0030000000000001,"15min":1.415},"min":{"1min":0.38700000000000001,"5min":0.371,"15min":0.371},"max":{"1min":5.1710000000000003,"5min":9.7029999999999994,"15min":9.7029999999999994},"last":0.86199999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795121,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1624348,"kb_used_data":1448276,"kb_used_omap":11,"kb_used_meta":176052,"kb_avail":92120804,"statfs":{"total":95995035648,"available":94331703296,"internally_reserved":0,"allocated":1483034624,"data_stored":1482415491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":180277290},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.3859999999999999,"15min":0.91800000000000004},"min":{"1min":0.217,"5min":0.191,"15min":0.191},"max":{"1min":6.8609999999999998,"5min":7.577,"15min":7.577},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.341,"5min":1.5,"15min":0.93400000000000005},"min":{"1min":0.27700000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.649,"5min":8.3529999999999998,"15min":8.3529999999999998},"last":0.94399999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.3799999999999999,"15min":0.94599999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.1669999999999998,"5min":8.1669999999999998,"15min":8.1669999999999998},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.232,"5min":1.6619999999999999,"15min":1.0289999999999999},"min":{"1min":0.36399999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.569,"5min":11.550000000000001,"15min":11.550000000000001},"last":0.98399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.1819999999999999,"15min":0.93799999999999994},"min":{"1min":0.35399999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.6440000000000001,"5min":10.044,"15min":10.044},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.76900000000000002,"5min":1.409,"15min":1.036},"min":{"1min":0.437,"5min":0.25,"15min":0.25},"max":{"1min":2.9500000000000002,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":1.0509999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":1.6639999999999999,"15min":1.1879999999999999},"min":{"1min":0.38600000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.0659999999999998,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":1.03},{"interface":"front","average":{"1min":1.22,"5min":1.915,"15min":1.2749999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.3799999999999999,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":5.4889999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0409999999999999,"15min":1.0309999999999999},"min":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.113,"5min":0.98499999999999999,"15min":0.96799999999999997},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":4.649}]},{"osd":6,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.8819999999999999,"15min":1.3029999999999999},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.4180000000000001,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":5.3949999999999996},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.2430000000000001,"15min":1.115},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":4.2629999999999999,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.83499999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.5309999999999999,"15min":1.26},"min":{"1min":0.40600000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.8230000000000004,"5min":8.2430000000000003,"15min":8.2430000000000003},"last":2.242},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.6559999999999999,"15min":1.276},"min":{"1min":0.40000000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.0259999999999998,"5min":8.952,"15min":8.952},"last":1.01}]}]},{"osd":3,"up_from":103,"seq":442381631592,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1645212,"kb_used_data":1537300,"kb_used_omap":5,"kb_used_meta":107898,"kb_avail":92099940,"statfs":{"total":95995035648,"available":94310338560,"internally_reserved":0,"allocated":1574195200,"data_stored":1573575723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":110487726},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":2.2959999999999998,"15min":1.2649999999999999},"min":{"1min":0.29799999999999999,"5min":0.187,"15min":0.187},"max":{"1min":1.03,"5min":61.966999999999999,"15min":61.966999999999999},"last":2.0609999999999999},{"interface":"front","average":{"1min":1.113,"5min":2.3039999999999998,"15min":1.2350000000000001},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.7320000000000002,"5min":62.052999999999997,"15min":62.052999999999997},"last":5.4660000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.806,"15min":1.073},"min":{"1min":0.309,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6179999999999999,"5min":61.395000000000003,"15min":61.395000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.7210000000000001,"15min":1.1120000000000001},"min":{"1min":0.39000000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":1.3380000000000001,"5min":61.179000000000002,"15min":61.179000000000002},"last":1.1839999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":3.7170000000000001,"15min":1.7749999999999999},"min":{"1min":0.29099999999999998,"5min":0.25,"15min":0.25},"max":{"1min":1.2849999999999999,"5min":181.959,"15min":181.959},"last":3.1200000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":3.621,"15min":1.8260000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.93999999999999995,"5min":181.91200000000001,"15min":181.91200000000001},"last":3.0600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.98199999999999998,"15min":1.034},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.165,"5min":6.0190000000000001,"15min":6.0190000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.1839999999999999,"15min":1.052},"min":{"1min":0.379,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.181,"5min":7.0739999999999998,"15min":7.0739999999999998},"last":5.3810000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.96499999999999997,"15min":1.0229999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":1.917},{"interface":"front","average":{"1min":0.872,"5min":0.997,"15min":1.056},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.996,"5min":1.996,"15min":1.996},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.01,"15min":1.0249999999999999},"min":{"1min":0.42199999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.3019999999999996,"5min":7.1120000000000001,"15min":7.1120000000000001},"last":5.4279999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.0149999999999999,"15min":1.0609999999999999},"min":{"1min":0.48899999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.6259999999999999,"5min":6.0039999999999996,"15min":6.0039999999999996},"last":0.44}]},{"osd":7,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.119,"15min":1.123},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.681,"5min":6.0359999999999996,"15min":6.0359999999999996},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.877,"5min":1.1319999999999999,"15min":1.177},"min":{"1min":0.57199999999999995,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.3819999999999999,"5min":6.8209999999999997,"15min":6.8209999999999997},"last":0.60299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019194,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1189932,"kb_used_data":892708,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92555220,"statfs":{"total":95995035648,"available":94776545280,"internally_reserved":0,"allocated":914132992,"data_stored":913615983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.1669999999999998,"15min":1.294},"min":{"1min":0.26800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.4269999999999996,"5min":48.450000000000003,"15min":48.450000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.8240000000000001,"5min":2.3839999999999999,"15min":1.3919999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.4909999999999997,"5min":49.283000000000001,"15min":49.283000000000001},"last":0.75800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.5980000000000001,"15min":1.131},"min":{"1min":0.32200000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.6059999999999999,"5min":8.8480000000000008,"15min":8.8480000000000008},"last":1.468},{"interface":"front","average":{"1min":1.349,"5min":1.8779999999999999,"15min":1.2190000000000001},"min":{"1min":0.377,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.2309999999999999,"5min":9.9399999999999995,"15min":9.9399999999999995},"last":0.55200000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:31:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":2.0609999999999999,"15min":1.306},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":9.2710000000000008,"5min":48.323,"15min":48.323},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.157,"5min":2.5390000000000001,"15min":1.486},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.311},"max":{"1min":9.0259999999999998,"5min":48.289000000000001,"15min":48.289000000000001},"last":0.73499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.1060000000000001,"15min":1.0549999999999999},"min":{"1min":0.36499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":2.0699999999999998,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.3520000000000001,"15min":1.1679999999999999},"min":{"1min":0.434,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.8550000000000004,"5min":11.112,"15min":11.112},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:27:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98999999999999999,"15min":0.95399999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.087,"15min":0.95399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":1.4770000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.996,"15min":0.98399999999999999},"min":{"1min":0.32400000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.4790000000000001,"5min":5.3550000000000004,"15min":5.3550000000000004},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.45,"15min":1.181},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7569999999999997,"5min":15.977,"15min":15.977},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.492,"15min":1.252},"min":{"1min":0.34499999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8879999999999999,"5min":11.725,"15min":11.725},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.224,"15min":1.155},"min":{"1min":0.42599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.1219999999999999,"5min":6.5730000000000004,"15min":6.5730000000000004},"last":0.92800000000000005}]}]},{"osd":5,"up_from":36,"seq":154618822912,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3745056,"kb_used_data":3439640,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90000096,"statfs":{"total":95995035648,"available":92160098304,"internally_reserved":0,"allocated":3522191360,"data_stored":3521698716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8919999999999999,"5min":1.474,"15min":1.073},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":53.731000000000002,"5min":53.731000000000002,"15min":53.731000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":1.5429999999999999,"15min":1.0620000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":53.222999999999999,"5min":53.222999999999999,"15min":53.222999999999999},"last":2.359}]},{"osd":1,"last update":"Mon Oct 9 04:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.98099999999999998,"15min":0.997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.252},{"interface":"front","average":{"1min":1.038,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.94699999999999995,"15min":0.94099999999999995},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":2.4430000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":2.5390000000000001,"5min":2.5390000000000001,"15min":2.5390000000000001},"last":2.4649999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.008,"15min":0.99399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":2.2789999999999999,"5min":2.2789999999999999,"15min":2.2789999999999999},"last":0.378},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.1240000000000001,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5140000000000002,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":2.4180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:43 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96299999999999997,"15min":0.94899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9100000000000001,"5min":3.9100000000000001,"15min":3.9100000000000001},"last":5.6479999999999997},{"interface":"front","average":{"1min":1.304,"5min":1.0900000000000001,"15min":1.012},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":3.9990000000000001},"last":2.395}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.998,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.9849999999999999,"5min":3.9849999999999999,"15min":3.9849999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0740000000000001,"15min":1.0660000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.847,"5min":1.847,"15min":2.5840000000000001},"last":0.32800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1379999999999999,"15min":1.113},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.431,"5min":3.992,"15min":3.992},"last":0.312},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.1120000000000001,"15min":1.095},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":1.4610000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":745734144,"data_stored":745734144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":802160640,"data_stored":802160640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1489371136,"data_stored":1489371136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1606287360,"data_stored":1606287360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":666894336,"data_stored":666894336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1008664576,"data_stored":1008664576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":997326848,"data_stored":997326848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:32:38.992 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:32:38.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-09T04:32:39.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:39.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:39.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:39.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:39.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:39.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:39.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:39.605 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:39.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:39 smithi082 bash[77063]: cluster 2023-10-09T04:32:39.522741+0000 mon.a (mon.0) 801 : cluster 3 Health check update: Degraded data redundancy: 15010/141597 objects degraded (10.601%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:39.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:39] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T04:32:39.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:39 smithi082 bash[79733]: cluster 2023-10-09T04:32:39.522741+0000 mon.a (mon.0) 801 : cluster 3 Health check update: Degraded data redundancy: 15010/141597 objects degraded (10.601%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:39.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:39.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:39.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 54322 54306 113.108 80.6875 0.00572306 0.0088354 2023-10-09T04:32:39.877 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.906 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:39.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:39 smithi153 bash[59565]: cluster 2023-10-09T04:32:39.522741+0000 mon.a (mon.0) 801 : cluster 3 Health check update: Degraded data redundancy: 15010/141597 objects degraded (10.601%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:40.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:40.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:40.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:40.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:40.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:40.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:40.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:40.452 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:40.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:40.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:40.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:40.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:40.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:40.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:40.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 55651 55635 112.139 83.0625 0.00901619 0.00891284 2023-10-09T04:32:40.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:40.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:40.881 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:32:40.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:40 smithi082 bash[77063]: cluster 2023-10-09T04:32:39.496249+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v511: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 15812/150081 objects degraded (10.536%) 2023-10-09T04:32:40.890 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:40 smithi082 bash[79733]: cluster 2023-10-09T04:32:39.496249+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v511: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 15812/150081 objects degraded (10.536%) 2023-10-09T04:32:40.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:40 smithi153 bash[59565]: cluster 2023-10-09T04:32:39.496249+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v511: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 15812/150081 objects degraded (10.536%) 2023-10-09T04:32:40.966 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:40.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:41.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:32:41.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:41.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:41.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:41.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:41.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:41.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:41.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:41.649 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:41.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:41.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:41.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 56844 56828 110.964 74.5625 0.00776273 0.00900601 2023-10-09T04:32:41.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:41.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:41.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:42.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:42.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:42.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:42.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:42.346 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:42.450 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:42.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:42.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:42.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:42 smithi082 bash[77063]: cluster 2023-10-09T04:32:41.497265+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v512: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 17098/161184 objects degraded (10.608%) 2023-10-09T04:32:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:42 smithi082 bash[79733]: cluster 2023-10-09T04:32:41.497265+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v512: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 17098/161184 objects degraded (10.608%) 2023-10-09T04:32:42.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:42.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:42.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 58174 58158 110.121 83.125 0.0156867 0.0090752 2023-10-09T04:32:42.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:42.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:42.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:42 smithi153 bash[59565]: cluster 2023-10-09T04:32:41.497265+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v512: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 17098/161184 objects degraded (10.608%) 2023-10-09T04:32:42.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:43.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:43.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:43.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:43.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:43.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:43.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:43.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:43.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:43.503 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:43.512 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:43.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:43.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:43.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:43.709 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:43.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:43.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:43.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 59373 59357 109.086 74.9375 0.00369514 0.00916157 2023-10-09T04:32:43.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:43.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:43.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:44.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:44.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:44.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:44.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:44.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:44.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:44.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:44.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:44.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:44.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:44.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:44.760 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:44.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 60585 60569 108.133 75.75 0.0186593 0.00924204 2023-10-09T04:32:44.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:44.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[77063]: cluster 2023-10-09T04:32:43.498223+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v513: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 17098/165903 objects degraded (10.306%) 2023-10-09T04:32:44.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[77063]: audit 2023-10-09T04:32:44.271569+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:44.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[77063]: audit 2023-10-09T04:32:44.275646+0000 mon.a (mon.0) 803 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[77063]: cluster 2023-10-09T04:32:44.524028+0000 mon.a (mon.0) 804 : cluster 3 Health check update: Degraded data redundancy: 17098/165903 objects degraded (10.306%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[79733]: cluster 2023-10-09T04:32:43.498223+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v513: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 17098/165903 objects degraded (10.306%) 2023-10-09T04:32:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[79733]: audit 2023-10-09T04:32:44.271569+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[79733]: audit 2023-10-09T04:32:44.275646+0000 mon.a (mon.0) 803 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:44 smithi082 bash[79733]: cluster 2023-10-09T04:32:44.524028+0000 mon.a (mon.0) 804 : cluster 3 Health check update: Degraded data redundancy: 17098/165903 objects degraded (10.306%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:44.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:44.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:44 smithi153 bash[59565]: cluster 2023-10-09T04:32:43.498223+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v513: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s; 17098/165903 objects degraded (10.306%) 2023-10-09T04:32:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:44 smithi153 bash[59565]: audit 2023-10-09T04:32:44.271569+0000 mon.a (mon.0) 802 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:32:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:44 smithi153 bash[59565]: audit 2023-10-09T04:32:44.275646+0000 mon.a (mon.0) 803 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:44 smithi153 bash[59565]: cluster 2023-10-09T04:32:44.524028+0000 mon.a (mon.0) 804 : cluster 3 Health check update: Degraded data redundancy: 17098/165903 objects degraded (10.306%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:44.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:45.126 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:45.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:45.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:45.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:45.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:45.422 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:45.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.541 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:45.544 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:45.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:45.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.682 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:45.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:45.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:45.859 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 61838 61822 107.305 78.3125 0.0180298 0.0093143 2023-10-09T04:32:45.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:45.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:45.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:45.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:46.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:46.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:46.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:46.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:46.324 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:46.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:46.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:46.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:46.487 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:46.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:46.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:46.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:46.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:46 smithi082 bash[77063]: cluster 2023-10-09T04:32:45.499152+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v514: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:46.745 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:46 smithi082 bash[79733]: cluster 2023-10-09T04:32:45.499152+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v514: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:46.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:46.860 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 63102 63086 106.54 79 0.00449503 0.00938011 2023-10-09T04:32:46.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:46.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:46.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:46.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:46 smithi153 bash[59565]: cluster 2023-10-09T04:32:45.499152+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v514: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:46.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:47.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:47.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.155 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:47.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:47.283 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:47.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:47.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:47.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:47.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:47.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:47.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:47.860 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 64326 64310 105.749 76.5 0.00914131 0.00945025 2023-10-09T04:32:47.919 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:47.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:48.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:48.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:48.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:48.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:48.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:48.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:48.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:48.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:48.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:48.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:48.853 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:48.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 65543 65527 104.986 76.0625 0.0230137 0.00951983 2023-10-09T04:32:48.881 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:48.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:48 smithi153 bash[59565]: cluster 2023-10-09T04:32:47.499672+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v515: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:48.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:48 smithi082 bash[77063]: cluster 2023-10-09T04:32:47.499672+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v515: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:48.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:48 smithi082 bash[79733]: cluster 2023-10-09T04:32:47.499672+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v515: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 19013/180084 objects degraded (10.558%) 2023-10-09T04:32:48.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:49.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:49.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:49.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:49.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:49.389 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:49.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:49.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:49.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:49.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.680 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:49.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:49 smithi082 bash[77063]: cluster 2023-10-09T04:32:49.525027+0000 mon.a (mon.0) 805 : cluster 3 Health check update: Degraded data redundancy: 19013/180084 objects degraded (10.558%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:49.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:49] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T04:32:49.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:49 smithi082 bash[79733]: cluster 2023-10-09T04:32:49.525027+0000 mon.a (mon.0) 805 : cluster 3 Health check update: Degraded data redundancy: 19013/180084 objects degraded (10.558%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:49.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:49.860 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:32:49.860240+0000 min lat: 0.00116931 max lat: 0.243754 avg lat: 0.00958896 2023-10-09T04:32:49.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:32:49.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 66736 66720 104.226 74.5625 0.0047979 0.00958896 2023-10-09T04:32:49.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:49 smithi153 bash[59565]: cluster 2023-10-09T04:32:49.525027+0000 mon.a (mon.0) 805 : cluster 3 Health check update: Degraded data redundancy: 19013/180084 objects degraded (10.558%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:49.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:49.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:50.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:50.096 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:50.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:50.199 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:50.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:50.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:50.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:50.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:50.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:50.492 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:50.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:50.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:50.667 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:50 smithi082 bash[77063]: cluster 2023-10-09T04:32:49.500594+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v516: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 19784/188121 objects degraded (10.517%) 2023-10-09T04:32:50.667 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:50 smithi082 bash[79733]: cluster 2023-10-09T04:32:49.500594+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v516: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 19784/188121 objects degraded (10.517%) 2023-10-09T04:32:50.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:50.760 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:50.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 67896 67880 103.452 72.5 0.0113067 0.00966084 2023-10-09T04:32:50.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:50.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:50.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:50.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:50.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:50 smithi153 bash[59565]: cluster 2023-10-09T04:32:49.500594+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v516: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 19784/188121 objects degraded (10.517%) 2023-10-09T04:32:50.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:51.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:51.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:51.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:51.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:51.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:51.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:51.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:51.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:51.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:51.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:51.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 68946 68930 102.551 65.625 0.00894789 0.00974619 2023-10-09T04:32:51.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:51.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:52.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:52.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:52.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:52.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:52.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:52.335 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:52.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:52.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:52.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:52.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:52.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:52.710 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:52 smithi153 bash[59565]: cluster 2023-10-09T04:32:51.501495+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v517: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 20972/198630 objects degraded (10.558%) 2023-10-09T04:32:52.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:52.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:52 smithi082 bash[77063]: cluster 2023-10-09T04:32:51.501495+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v517: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 20972/198630 objects degraded (10.558%) 2023-10-09T04:32:52.791 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:52 smithi082 bash[79733]: cluster 2023-10-09T04:32:51.501495+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v517: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 20972/198630 objects degraded (10.558%) 2023-10-09T04:32:52.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:52.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:52.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 70053 70037 101.775 69.1875 0.018874 0.00982074 2023-10-09T04:32:52.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:52.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:52.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:52.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:53.094 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:53.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:53.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:53.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:53.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:53.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:53.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:53.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:53.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:53.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:53.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 71230 71214 101.134 73.5625 0.0101783 0.00988252 2023-10-09T04:32:53.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:53.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:54.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:54.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.192 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:54.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:54.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.447 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:54.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:54.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:54.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:54 smithi082 bash[77063]: cluster 2023-10-09T04:32:53.502292+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v518: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 20972/202845 objects degraded (10.339%) 2023-10-09T04:32:54.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:54 smithi082 bash[77063]: cluster 2023-10-09T04:32:54.527105+0000 mon.a (mon.0) 806 : cluster 3 Health check update: Degraded data redundancy: 20972/202845 objects degraded (10.339%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:54.689 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:54 smithi082 bash[79733]: cluster 2023-10-09T04:32:53.502292+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v518: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 20972/202845 objects degraded (10.339%) 2023-10-09T04:32:54.689 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:54 smithi082 bash[79733]: cluster 2023-10-09T04:32:54.527105+0000 mon.a (mon.0) 806 : cluster 3 Health check update: Degraded data redundancy: 20972/202845 objects degraded (10.339%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:54.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:54.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:54.808 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:54.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:54 smithi153 bash[59565]: cluster 2023-10-09T04:32:53.502292+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v518: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 20972/202845 objects degraded (10.339%) 2023-10-09T04:32:54.817 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:54 smithi153 bash[59565]: cluster 2023-10-09T04:32:54.527105+0000 mon.a (mon.0) 806 : cluster 3 Health check update: Degraded data redundancy: 20972/202845 objects degraded (10.339%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:54.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:54.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 72366 72350 100.464 71 0.0147686 0.00994725 2023-10-09T04:32:54.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:54.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:55.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:55.209 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:55.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:55.338 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:55.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:55.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:55.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:55.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:55.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 73560 73544 99.9024 74.625 0.011206 0.0100053 2023-10-09T04:32:55.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:55.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:55.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:55.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:56.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:56.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:56.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:56.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:56.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:56.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:56.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:56.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:56.518 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:56.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:56.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:56.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:56.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:56.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:56.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 16 74686 74670 99.2741 70.375 0.0310215 0.010068 2023-10-09T04:32:56.911 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:56.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:56.938 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:56 smithi153 bash[59565]: cluster 2023-10-09T04:32:55.503319+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v519: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:56.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:56.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:56.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:56 smithi082 bash[77063]: cluster 2023-10-09T04:32:55.503319+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v519: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:56.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:56 smithi082 bash[79733]: cluster 2023-10-09T04:32:55.503319+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v519: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:57.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:57.062 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:57.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:57.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:57.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:57.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:57.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:57.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:57.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:57.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:57.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:57.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:57.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:57.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:57.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:57.860 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:57.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 75736 75720 98.573 65.625 0.00787703 0.0101398 2023-10-09T04:32:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:57.963 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:57.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:57.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:57.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:58.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:58.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.266 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:58.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:58.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.395 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:58.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:58.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:58.555 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:58.555 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:58.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:58.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:58.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:58 smithi082 bash[77063]: cluster 2023-10-09T04:32:57.504059+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v520: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:58.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.807 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:58.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 76754 76738 97.8598 63.625 0.00598062 0.0102123 2023-10-09T04:32:58.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:58.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:58 smithi153 bash[59565]: cluster 2023-10-09T04:32:57.504059+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v520: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:58.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:58.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:58.940 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:58 smithi082 bash[79733]: cluster 2023-10-09T04:32:57.504059+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v520: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 22711/215499 objects degraded (10.539%) 2023-10-09T04:32:59.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:59.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:59.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:59.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:59.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:59.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:59.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:59.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:59.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:59.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:59.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 77764 77748 97.165 63.125 0.0255703 0.0102869 2023-10-09T04:32:59.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:59 smithi153 bash[59565]: audit 2023-10-09T04:32:59.264747+0000 mon.a (mon.0) 807 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:32:59 smithi153 bash[59565]: cluster 2023-10-09T04:32:59.531761+0000 mon.a (mon.0) 808 : cluster 3 Health check update: Degraded data redundancy: 22711/215499 objects degraded (10.539%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:59.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:59.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:59 smithi082 bash[77063]: audit 2023-10-09T04:32:59.264747+0000 mon.a (mon.0) 807 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:59.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:32:59 smithi082 bash[77063]: cluster 2023-10-09T04:32:59.531761+0000 mon.a (mon.0) 808 : cluster 3 Health check update: Degraded data redundancy: 22711/215499 objects degraded (10.539%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:59.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:59 smithi082 bash[79733]: audit 2023-10-09T04:32:59.264747+0000 mon.a (mon.0) 807 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:59.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:32:59 smithi082 bash[79733]: cluster 2023-10-09T04:32:59.531761+0000 mon.a (mon.0) 808 : cluster 3 Health check update: Degraded data redundancy: 22711/215499 objects degraded (10.539%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:32:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:32:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:32:59] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T04:32:59.990 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:59.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:00.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:00.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:00.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:00.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:00.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:33:00.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:00.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:00.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:00.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:00.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.770 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:00 smithi082 bash[79733]: cluster 2023-10-09T04:32:59.644026+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v521: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23383/222879 objects degraded (10.491%) 2023-10-09T04:33:00.771 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:00 smithi082 bash[77063]: cluster 2023-10-09T04:32:59.644026+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v521: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23383/222879 objects degraded (10.491%) 2023-10-09T04:33:00.776 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:33:00.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:00.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:00.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:33:00.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 78535 78519 96.2047 48.1875 0.0379192 0.0103875 2023-10-09T04:33:00.864 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:33:00.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:00.917 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:33:00.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:00.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:00.960 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:33:00.997 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:33:01.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:01.039 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:33:01.040 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.040 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.040 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:33:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:33:01.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:00 smithi153 bash[59565]: cluster 2023-10-09T04:32:59.644026+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v521: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23383/222879 objects degraded (10.491%) 2023-10-09T04:33:01.055 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T04:33:01.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:01.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:01.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:01.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:01.322 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:01.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:01.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:01.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:01.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:01.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:01.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:01.736 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:33:01.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:01.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:01 smithi153 bash[59565]: audit 2023-10-09T04:33:00.721469+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.82:0/1363121494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:01.841 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:01 smithi153 bash[59565]: audit 2023-10-09T04:33:00.961012+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.82:0/2535524690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.842 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:01 smithi153 bash[59565]: audit 2023-10-09T04:33:00.961820+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.842 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:01 smithi153 bash[59565]: audit 2023-10-09T04:33:01.050829+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.82:0/990916842' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:33:01.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:01.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:01.861 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 79471 79455 95.4796 58.5 0.0177876 0.0104679 2023-10-09T04:33:01.962 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:01.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[77063]: audit 2023-10-09T04:33:00.721469+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.82:0/1363121494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:01.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[77063]: audit 2023-10-09T04:33:00.961012+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.82:0/2535524690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[77063]: audit 2023-10-09T04:33:00.961820+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[77063]: audit 2023-10-09T04:33:01.050829+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.82:0/990916842' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:33:01.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[79733]: audit 2023-10-09T04:33:00.721469+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.82:0/1363121494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:01.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[79733]: audit 2023-10-09T04:33:00.961012+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.82:0/2535524690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[79733]: audit 2023-10-09T04:33:00.961820+0000 mon.a (mon.0) 809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:01.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:01 smithi082 bash[79733]: audit 2023-10-09T04:33:01.050829+0000 mon.a (mon.0) 810 : audit 1 from='client.? 172.21.15.82:0/990916842' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:33:01.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:01.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:02.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:02.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:02.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:02.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:02.258 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:02.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:02.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:02.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:02.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:02.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:02.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:02.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:02.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:02.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:02.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:02.862 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 80434 80418 94.8137 60.1875 0.00656406 0.0105419 2023-10-09T04:33:02.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.649664+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v522: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 24369/231777 objects degraded (10.514%) 2023-10-09T04:33:02.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.692979+0000 mon.a (mon.0) 811 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.715203+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.719524+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.721012+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: audit 2023-10-09T04:33:01.722141+0000 mon.a (mon.0) 812 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.722233+0000 mon.a (mon.0) 813 : cluster 0 osdmap e138: 8 total, 7 up, 8 in 2023-10-09T04:33:02.882 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[77063]: cluster 2023-10-09T04:33:01.724394+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.649664+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v522: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 24369/231777 objects degraded (10.514%) 2023-10-09T04:33:02.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.692979+0000 mon.a (mon.0) 811 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:02.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.715203+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.719524+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.721012+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.884 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: audit 2023-10-09T04:33:01.722141+0000 mon.a (mon.0) 812 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:02.884 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.722233+0000 mon.a (mon.0) 813 : cluster 0 osdmap e138: 8 total, 7 up, 8 in 2023-10-09T04:33:02.884 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:02 smithi082 bash[79733]: cluster 2023-10-09T04:33:01.724394+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.649664+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v522: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 24369/231777 objects degraded (10.514%) 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.692979+0000 mon.a (mon.0) 811 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.715203+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.719524+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.721012+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: audit 2023-10-09T04:33:01.722141+0000 mon.a (mon.0) 812 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.722233+0000 mon.a (mon.0) 813 : cluster 0 osdmap e138: 8 total, 7 up, 8 in 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:02 smithi153 bash[59565]: cluster 2023-10-09T04:33:01.724394+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T04:33:02.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:02.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:03.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:03.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:03.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:03.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:03.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:03.339 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:03.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:03.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:03.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:03.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:03.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:03.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:03.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:03.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:03.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:03.862 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 81518 81502 94.3124 67.75 0.00627161 0.0105975 2023-10-09T04:33:03.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:03.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:33:03.981 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:33:03.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:04.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:04.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:04.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.151 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:04.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T04:33:04.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:04.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:04.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:04.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:04.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:04.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.793 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:04 smithi082 bash[77063]: cluster 2023-10-09T04:33:03.650821+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v524: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 24369/235401 objects degraded (10.352%) 2023-10-09T04:33:04.794 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:04 smithi082 bash[77063]: cluster 2023-10-09T04:33:04.533763+0000 mon.a (mon.0) 814 : cluster 3 Health check update: Degraded data redundancy: 24369/235401 objects degraded (10.352%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:04.794 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:04 smithi082 bash[79733]: cluster 2023-10-09T04:33:03.650821+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v524: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 24369/235401 objects degraded (10.352%) 2023-10-09T04:33:04.794 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:04 smithi082 bash[79733]: cluster 2023-10-09T04:33:04.533763+0000 mon.a (mon.0) 814 : cluster 3 Health check update: Degraded data redundancy: 24369/235401 objects degraded (10.352%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:04.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:04.862 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 82586 82570 93.8112 66.75 0.00778509 0.0106538 2023-10-09T04:33:04.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:04.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:04.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:05.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:05.049 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:04 smithi153 bash[59565]: cluster 2023-10-09T04:33:03.650821+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v524: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 24369/235401 objects degraded (10.352%) 2023-10-09T04:33:05.049 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:04 smithi153 bash[59565]: cluster 2023-10-09T04:33:04.533763+0000 mon.a (mon.0) 814 : cluster 3 Health check update: Degraded data redundancy: 24369/235401 objects degraded (10.352%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:05.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:05.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:05.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:05.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:05.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:05.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:05.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:05.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[77063]: audit 2023-10-09T04:33:05.093839+0000 mon.a (mon.0) 815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[77063]: audit 2023-10-09T04:33:05.569684+0000 mon.a (mon.0) 816 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[77063]: audit 2023-10-09T04:33:05.591217+0000 mon.a (mon.0) 817 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[77063]: audit 2023-10-09T04:33:05.626426+0000 mon.a (mon.0) 818 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[77063]: audit 2023-10-09T04:33:05.640859+0000 mon.a (mon.0) 819 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[79733]: audit 2023-10-09T04:33:05.093839+0000 mon.a (mon.0) 815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[79733]: audit 2023-10-09T04:33:05.569684+0000 mon.a (mon.0) 816 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[79733]: audit 2023-10-09T04:33:05.591217+0000 mon.a (mon.0) 817 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.722 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[79733]: audit 2023-10-09T04:33:05.626426+0000 mon.a (mon.0) 818 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:05 smithi082 bash[79733]: audit 2023-10-09T04:33:05.640859+0000 mon.a (mon.0) 819 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:05.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:05.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:05.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 83498 83482 93.1428 57 0.0121508 0.0107286 2023-10-09T04:33:05.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:05.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:05.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:33:06.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:05 smithi153 bash[59565]: audit 2023-10-09T04:33:05.093839+0000 mon.a (mon.0) 815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:06.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:05 smithi153 bash[59565]: audit 2023-10-09T04:33:05.569684+0000 mon.a (mon.0) 816 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:06.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:05 smithi153 bash[59565]: audit 2023-10-09T04:33:05.591217+0000 mon.a (mon.0) 817 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:06.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:05 smithi153 bash[59565]: audit 2023-10-09T04:33:05.626426+0000 mon.a (mon.0) 818 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:06.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:05 smithi153 bash[59565]: audit 2023-10-09T04:33:05.640859+0000 mon.a (mon.0) 819 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:06.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:33:06.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:06.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:06.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:06.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:06.825 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:06.850 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:06.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 84382 84366 92.4781 55.25 0.0310353 0.0108079 2023-10-09T04:33:06.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:06.965 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:06 smithi082 bash[77063]: cluster 2023-10-09T04:33:05.651768+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v525: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:06.966 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:06 smithi082 bash[79733]: cluster 2023-10-09T04:33:05.651768+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v525: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:06.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:06.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:07.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:07.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:07.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:06 smithi153 bash[59565]: cluster 2023-10-09T04:33:05.651768+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v525: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:07.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:07.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:07.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:07.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:07.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:07.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:07.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:07.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:07.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 85388 85372 91.9677 62.875 0.0127459 0.0108678 2023-10-09T04:33:07.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:07.894 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:07.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:07.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:08.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:08.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:08.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:08.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:08.321 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:08.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:08.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:08.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:08.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:08.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:08.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:08.728 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:08 smithi082 bash[77063]: cluster 2023-10-09T04:33:07.652466+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v526: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:08.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:08 smithi082 bash[79733]: cluster 2023-10-09T04:33:07.652466+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v526: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:08.731 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:08.759 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:08.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:08.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:08.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 86356 86340 91.4343 60.5 0.00465482 0.010931 2023-10-09T04:33:08.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:08.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:08.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:09.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:09.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:09.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:08 smithi153 bash[59565]: cluster 2023-10-09T04:33:07.652466+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v526: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25952/246402 objects degraded (10.532%) 2023-10-09T04:33:09.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:09.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:09.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:09.346 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:33:09.346 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:33:09.346 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 6, 7] 2023-10-09T04:33:09.346 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T04:33:09.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T04:33:09.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:09.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:09.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:09.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:09.676 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:09 smithi153 systemd[1]: Stopping Ceph osd.6 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:33:09.676 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:09 smithi153 bash[39643]: debug 2023-10-09T04:33:09.512+0000 7f099cef3700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:33:09.676 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:09 smithi153 bash[39643]: debug 2023-10-09T04:33:09.512+0000 7f099cef3700 -1 osd.6 138 *** Got signal Terminated *** 2023-10-09T04:33:09.676 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:09 smithi153 bash[39643]: debug 2023-10-09T04:33:09.512+0000 7f099cef3700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:33:09.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:09.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:09.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:09.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:09.838 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:09] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-09T04:33:09.838 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:09 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.518169+0000 mon.a (mon.0) 820 : cluster 1 osd.6 marked itself down and dead 2023-10-09T04:33:09.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:09 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.535123+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 25952/246402 objects degraded (10.532%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:09.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:09 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.518169+0000 mon.a (mon.0) 820 : cluster 1 osd.6 marked itself down and dead 2023-10-09T04:33:09.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:09 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.535123+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 25952/246402 objects degraded (10.532%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:09.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:09.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:33:09.868750+0000 min lat: 0.00116931 max lat: 0.243754 avg lat: 0.0109841 2023-10-09T04:33:09.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:33:09.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 87257 87241 90.849 56.3125 0.00565059 0.0109841 2023-10-09T04:33:09.929 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:09.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:09.960 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:09 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.518169+0000 mon.a (mon.0) 820 : cluster 1 osd.6 marked itself down and dead 2023-10-09T04:33:09.960 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:09 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.535123+0000 mon.a (mon.0) 821 : cluster 3 Health check update: Degraded data redundancy: 25952/246402 objects degraded (10.532%), 12 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:09.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:09.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:10.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:10.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:10.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:10.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:10.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:10.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:10.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.610 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:10.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:10.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:10.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:10.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:10.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:10.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:10.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:10.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 87328 87312 89.4327 4.4375 0.00506489 0.0110858 2023-10-09T04:33:10.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.653335+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v527: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 26559/252732 objects degraded (10.509%) 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.726125+0000 mon.a (mon.0) 822 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.741857+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.744723+0000 mon.a (mon.0) 823 : cluster 0 osdmap e139: 8 total, 6 up, 8 in 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.744785+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.744975+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[77063]: cluster 2023-10-09T04:33:09.746649+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.653335+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v527: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 26559/252732 objects degraded (10.509%) 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.726125+0000 mon.a (mon.0) 822 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.741857+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.744723+0000 mon.a (mon.0) 823 : cluster 0 osdmap e139: 8 total, 6 up, 8 in 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.744785+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.744975+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:10 smithi082 bash[79733]: cluster 2023-10-09T04:33:09.746649+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.653335+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v527: 89 pgs: 1 active+clean+remapped, 12 active+undersized+degraded, 14 active+undersized, 62 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 26559/252732 objects degraded (10.509%) 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.726125+0000 mon.a (mon.0) 822 : cluster 3 Health check update: 2 osds down (OSD_DOWN) 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.741857+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.744723+0000 mon.a (mon.0) 823 : cluster 0 osdmap e139: 8 total, 6 up, 8 in 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.744785+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.744975+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:10 smithi153 bash[59565]: cluster 2023-10-09T04:33:09.746649+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:33:10.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:11.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:11.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:11.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:11.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:11.381 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:11.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:11.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:11.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:11.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:11.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 87335 87319 87.9975 0.4375 0.00408837 0.0110854 2023-10-09T04:33:11.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:11.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:11.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: cluster 2023-10-09T04:33:10.742200+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: cluster 2023-10-09T04:33:10.744195+0000 mon.a (mon.0) 824 : cluster 0 osdmap e140: 8 total, 6 up, 8 in 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: cluster 2023-10-09T04:33:10.745627+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: cluster 2023-10-09T04:33:10.749874+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: audit 2023-10-09T04:33:10.988934+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[77063]: audit 2023-10-09T04:33:10.997494+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: cluster 2023-10-09T04:33:10.742200+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: cluster 2023-10-09T04:33:10.744195+0000 mon.a (mon.0) 824 : cluster 0 osdmap e140: 8 total, 6 up, 8 in 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: cluster 2023-10-09T04:33:10.745627+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: cluster 2023-10-09T04:33:10.749874+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: audit 2023-10-09T04:33:10.988934+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:11 smithi082 bash[79733]: audit 2023-10-09T04:33:10.997494+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:12.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:12.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: cluster 2023-10-09T04:33:10.742200+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:12.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: cluster 2023-10-09T04:33:10.744195+0000 mon.a (mon.0) 824 : cluster 0 osdmap e140: 8 total, 6 up, 8 in 2023-10-09T04:33:12.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: cluster 2023-10-09T04:33:10.745627+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:12.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: cluster 2023-10-09T04:33:10.749874+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T04:33:12.071 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: audit 2023-10-09T04:33:10.988934+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:12.072 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:11 smithi153 bash[59565]: audit 2023-10-09T04:33:10.997494+0000 mon.a (mon.0) 826 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:12.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:12.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:12.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:12.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:12.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:12.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:12.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:12.410 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:12 smithi153 bash[79330]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T04:33:12.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:12.462 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:12.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:12.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:12.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:12.675 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:12 smithi153 bash[79496]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T04:33:12.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:12.744 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:12 smithi082 bash[77063]: cluster 2023-10-09T04:33:11.654064+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v530: 89 pgs: 18 peering, 3 stale+active+undersized+degraded, 46 active+clean, 9 active+undersized, 7 active+undersized+degraded, 1 active+clean+remapped, 2 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 26863/258564 objects degraded (10.389%) 2023-10-09T04:33:12.745 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:12 smithi082 bash[77063]: cluster 2023-10-09T04:33:11.995542+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:12.798 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:12 smithi153 bash[59565]: cluster 2023-10-09T04:33:11.654064+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v530: 89 pgs: 18 peering, 3 stale+active+undersized+degraded, 46 active+clean, 9 active+undersized, 7 active+undersized+degraded, 1 active+clean+remapped, 2 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 26863/258564 objects degraded (10.389%) 2023-10-09T04:33:12.806 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:12 smithi153 bash[59565]: cluster 2023-10-09T04:33:11.995542+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:12.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:12.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 87335 87319 86.601 0 - 0.0110854 2023-10-09T04:33:12.906 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:12 smithi082 bash[79733]: cluster 2023-10-09T04:33:11.654064+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v530: 89 pgs: 18 peering, 3 stale+active+undersized+degraded, 46 active+clean, 9 active+undersized, 7 active+undersized+degraded, 1 active+clean+remapped, 2 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 26863/258564 objects degraded (10.389%) 2023-10-09T04:33:12.906 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:12 smithi082 bash[79733]: cluster 2023-10-09T04:33:11.995542+0000 mon.a (mon.0) 827 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:12.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:12.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:13.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:13.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:13.177 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:13.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:13.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:13.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:13.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:13.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:13.567 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:13.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:13.692 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:13.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:13.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:13.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:13.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:13.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:13.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 87335 87319 85.248 0 - 0.0110854 2023-10-09T04:33:13.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:13.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:13.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:14.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:14.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:14.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:14.218 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:14.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:14.347 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:14.450 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:14.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:14.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:14.713 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:14.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:14.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T04:33:14.842 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:14.846 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:33:14.847 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T04:33:14.862 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:14.869 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 87335 87319 83.9368 0 - 0.0110854 2023-10-09T04:33:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: cluster 2023-10-09T04:33:13.655471+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v531: 89 pgs: 1 active+clean+remapped, 18 peering, 1 stale+active+undersized+degraded, 44 active+clean, 11 active+undersized, 2 undersized+degraded+peered, 8 active+undersized+degraded, 1 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 38195/260886 objects degraded (14.640%) 2023-10-09T04:33:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: audit 2023-10-09T04:33:14.249304+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: audit 2023-10-09T04:33:14.256110+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: audit 2023-10-09T04:33:14.271156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: audit 2023-10-09T04:33:14.272433+0000 mon.a (mon.0) 831 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:14 smithi153 bash[59565]: cluster 2023-10-09T04:33:14.536052+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 38195/260886 objects degraded (14.640%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:14.966 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:14.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:14.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: cluster 2023-10-09T04:33:13.655471+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v531: 89 pgs: 1 active+clean+remapped, 18 peering, 1 stale+active+undersized+degraded, 44 active+clean, 11 active+undersized, 2 undersized+degraded+peered, 8 active+undersized+degraded, 1 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 38195/260886 objects degraded (14.640%) 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: audit 2023-10-09T04:33:14.249304+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: audit 2023-10-09T04:33:14.256110+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: audit 2023-10-09T04:33:14.271156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: audit 2023-10-09T04:33:14.272433+0000 mon.a (mon.0) 831 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.991 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[79733]: cluster 2023-10-09T04:33:14.536052+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 38195/260886 objects degraded (14.640%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: cluster 2023-10-09T04:33:13.655471+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v531: 89 pgs: 1 active+clean+remapped, 18 peering, 1 stale+active+undersized+degraded, 44 active+clean, 11 active+undersized, 2 undersized+degraded+peered, 8 active+undersized+degraded, 1 stale+active+clean, 3 stale+active+undersized; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 38195/260886 objects degraded (14.640%) 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: audit 2023-10-09T04:33:14.249304+0000 mon.a (mon.0) 828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: audit 2023-10-09T04:33:14.256110+0000 mon.a (mon.0) 829 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: audit 2023-10-09T04:33:14.271156+0000 mon.a (mon.0) 830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: audit 2023-10-09T04:33:14.272433+0000 mon.a (mon.0) 831 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.992 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:14 smithi082 bash[77063]: cluster 2023-10-09T04:33:14.536052+0000 mon.a (mon.0) 832 : cluster 3 Health check update: Degraded data redundancy: 38195/260886 objects degraded (14.640%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:15.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:15.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:15.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:15.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:15.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:15.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.381 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:15.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:15.610 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:15.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:15.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:15.766 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:15.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 87335 87319 82.6652 0 - 0.0110854 2023-10-09T04:33:15.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:15.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:15 smithi153 bash[59565]: audit 2023-10-09T04:33:14.777637+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:15 smithi153 bash[59565]: audit 2023-10-09T04:33:14.778674+0000 mon.a (mon.0) 834 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:15 smithi153 bash[59565]: cluster 2023-10-09T04:33:14.780933+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v532: 89 pgs: 1 active+clean+remapped, 18 peering, 38 active+clean, 3 undersized+peered, 17 active+undersized, 4 undersized+degraded+peered, 8 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 49436/261945 objects degraded (18.873%) 2023-10-09T04:33:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:15 smithi153 bash[59565]: audit 2023-10-09T04:33:14.785645+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:15.993 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:16.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[77063]: audit 2023-10-09T04:33:14.777637+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[77063]: audit 2023-10-09T04:33:14.778674+0000 mon.a (mon.0) 834 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[77063]: cluster 2023-10-09T04:33:14.780933+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v532: 89 pgs: 1 active+clean+remapped, 18 peering, 38 active+clean, 3 undersized+peered, 17 active+undersized, 4 undersized+degraded+peered, 8 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 49436/261945 objects degraded (18.873%) 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[77063]: audit 2023-10-09T04:33:14.785645+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[79733]: audit 2023-10-09T04:33:14.777637+0000 mon.a (mon.0) 833 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[79733]: audit 2023-10-09T04:33:14.778674+0000 mon.a (mon.0) 834 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:16.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[79733]: cluster 2023-10-09T04:33:14.780933+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v532: 89 pgs: 1 active+clean+remapped, 18 peering, 38 active+clean, 3 undersized+peered, 17 active+undersized, 4 undersized+degraded+peered, 8 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 49436/261945 objects degraded (18.873%) 2023-10-09T04:33:16.138 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:15 smithi082 bash[79733]: audit 2023-10-09T04:33:14.785645+0000 mon.a (mon.0) 835 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:16.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:16.254 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:16.376 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:33:16.377 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:33:16.377 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:33:16.377 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:33:16.377 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:33:16.391 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:16.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:16.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:16.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:16.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:16.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:16.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:16.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 87335 87319 81.4316 0 - 0.0110854 2023-10-09T04:33:16.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:16.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:16 smithi153 bash[59565]: cluster 2023-10-09T04:33:15.745916+0000 mon.a (mon.0) 836 : cluster 3 Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-10-09T04:33:16.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:16.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:16.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:17.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:17.073 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:17.107 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:16 smithi082 bash[77063]: cluster 2023-10-09T04:33:15.745916+0000 mon.a (mon.0) 836 : cluster 3 Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-10-09T04:33:17.108 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:16 smithi082 bash[79733]: cluster 2023-10-09T04:33:15.745916+0000 mon.a (mon.0) 836 : cluster 3 Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-10-09T04:33:17.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:17.208 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:17.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:17.334 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:17.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:17.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:17.563 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:17.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:17.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:17.727 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:17.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:17.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:17.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:17.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 87335 87319 80.2342 0 - 0.0110854 2023-10-09T04:33:17.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:17 smithi153 bash[59565]: cluster 2023-10-09T04:33:16.782274+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v533: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:17.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:17.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:17.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:18.093 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:18.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:17 smithi082 bash[77063]: cluster 2023-10-09T04:33:16.782274+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v533: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:18.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:17 smithi082 bash[79733]: cluster 2023-10-09T04:33:16.782274+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v533: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:18.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:18.221 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:18.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:18.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:18.375 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:18.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:18.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:33:18.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:18.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:18.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:18.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:18.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:18.760 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:18.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:18.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 16 87335 87319 79.0716 0 - 0.0110854 2023-10-09T04:33:18.871 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:18 smithi082 bash[77063]: cluster 2023-10-09T04:33:17.756228+0000 mon.a (mon.0) 837 : cluster 3 Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:33:18.871 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:18 smithi082 bash[79733]: cluster 2023-10-09T04:33:17.756228+0000 mon.a (mon.0) 837 : cluster 3 Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:33:18.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:18.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:18.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:18.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:19.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:19.096 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:33:19.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:19.119 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:33:19.127 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:18 smithi153 bash[59565]: cluster 2023-10-09T04:33:17.756228+0000 mon.a (mon.0) 837 : cluster 3 Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:33:19.141 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:33:19.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:19.163 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:33:19.188 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:33:19.214 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-09T04:33:19.215 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:19.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:19.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:19.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:19.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:19.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:19.774 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:19] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.33.4" 2023-10-09T04:33:19.797 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:19.847 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5, 6] live_osds: [0, 1, 2, 3, 4, 7] 2023-10-09T04:33:19.847 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:33:19.847 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T04:33:19.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:33:19.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 87335 87319 77.9422 0 - 0.0110854 2023-10-09T04:33:19.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:19.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:19.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:20.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:20.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:19 smithi153 bash[59565]: cluster 2023-10-09T04:33:18.783049+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v534: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:20.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:19 smithi153 bash[59565]: audit 2023-10-09T04:33:19.070328+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.82:0/1928790743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.042 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:19 smithi153 bash[59565]: cluster 2023-10-09T04:33:19.547794+0000 mon.a (mon.0) 839 : cluster 3 Health check update: Degraded data redundancy: 60186/262044 objects degraded (22.968%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:20.062 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:20.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[79733]: cluster 2023-10-09T04:33:18.783049+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v534: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[79733]: audit 2023-10-09T04:33:19.070328+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.82:0/1928790743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[79733]: cluster 2023-10-09T04:33:19.547794+0000 mon.a (mon.0) 839 : cluster 3 Health check update: Degraded data redundancy: 60186/262044 objects degraded (22.968%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[77063]: cluster 2023-10-09T04:33:18.783049+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v534: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[77063]: audit 2023-10-09T04:33:19.070328+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.82:0/1928790743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:19 smithi082 bash[77063]: cluster 2023-10-09T04:33:19.547794+0000 mon.a (mon.0) 839 : cluster 3 Health check update: Degraded data redundancy: 60186/262044 objects degraded (22.968%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:33:20.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:20.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:20.326 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:20.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:20.452 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:20.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:20.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:20.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:20.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:20.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:20.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:20.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 16 87335 87319 76.8446 0 - 0.0110854 2023-10-09T04:33:20.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:20.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:20.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:21.086 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:21.170 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:33:21.171 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:33:21.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:21.247 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:21.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:21.382 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:21.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:21.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:21.754 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:21.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:21.858 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:33:21.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 16 87335 87319 75.7775 0 - 0.0110854 2023-10-09T04:33:21.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:21.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:21.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:21.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:22.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:22.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:22.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.163 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:21 smithi153 bash[59565]: cluster 2023-10-09T04:33:20.784519+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v535: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 7.2 MiB/s wr, 115 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:22.163 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:21 smithi153 bash[59565]: audit 2023-10-09T04:33:21.657374+0000 mon.a (mon.0) 840 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:22.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:21 smithi082 bash[77063]: cluster 2023-10-09T04:33:20.784519+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v535: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 7.2 MiB/s wr, 115 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:21 smithi082 bash[77063]: audit 2023-10-09T04:33:21.657374+0000 mon.a (mon.0) 840 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:21 smithi082 bash[79733]: cluster 2023-10-09T04:33:20.784519+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v535: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 7.2 MiB/s wr, 115 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:21 smithi082 bash[79733]: audit 2023-10-09T04:33:21.657374+0000 mon.a (mon.0) 840 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:22.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.319 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:22.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:22.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:22.595 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:22.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:22.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:22.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:22.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:22.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:22.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:22.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:22.870 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 87335 87319 74.7396 0 - 0.0110854 2023-10-09T04:33:22.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:22.979 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[77063]: cluster 2023-10-09T04:33:21.838602+0000 mon.a (mon.0) 841 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[77063]: cluster 2023-10-09T04:33:21.844426+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[77063]: audit 2023-10-09T04:33:21.850516+0000 mon.a (mon.0) 842 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[77063]: cluster 2023-10-09T04:33:21.850567+0000 mon.a (mon.0) 843 : cluster 0 osdmap e141: 8 total, 6 up, 8 in 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[77063]: cluster 2023-10-09T04:33:22.006955+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[79733]: cluster 2023-10-09T04:33:21.838602+0000 mon.a (mon.0) 841 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[79733]: cluster 2023-10-09T04:33:21.844426+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[79733]: audit 2023-10-09T04:33:21.850516+0000 mon.a (mon.0) 842 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[79733]: cluster 2023-10-09T04:33:21.850567+0000 mon.a (mon.0) 843 : cluster 0 osdmap e141: 8 total, 6 up, 8 in 2023-10-09T04:33:22.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:22 smithi082 bash[79733]: cluster 2023-10-09T04:33:22.006955+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:22.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:23.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:23.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:23.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:22 smithi153 bash[59565]: cluster 2023-10-09T04:33:21.838602+0000 mon.a (mon.0) 841 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:23.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:22 smithi153 bash[59565]: cluster 2023-10-09T04:33:21.844426+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:23.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:22 smithi153 bash[59565]: audit 2023-10-09T04:33:21.850516+0000 mon.a (mon.0) 842 : audit 1 from='client.? 172.21.15.82:0/1152363736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:23.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:22 smithi153 bash[59565]: cluster 2023-10-09T04:33:21.850567+0000 mon.a (mon.0) 843 : cluster 0 osdmap e141: 8 total, 6 up, 8 in 2023-10-09T04:33:23.116 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:22 smithi153 bash[59565]: cluster 2023-10-09T04:33:22.006955+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T04:33:23.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:23.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:33:23.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:23.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:23.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:23.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:23.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:23.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:23.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:23.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:23.665 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:23.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:23.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:23.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:23.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 87335 87319 73.7297 0 - 0.0110854 2023-10-09T04:33:23.905 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:23.918 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:23 smithi153 bash[59565]: cluster 2023-10-09T04:33:22.785842+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v537: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.6 MiB/s wr, 42 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:23.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:23.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:24.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:24.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:24.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:23 smithi082 bash[77063]: cluster 2023-10-09T04:33:22.785842+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v537: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.6 MiB/s wr, 42 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:24.187 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:23 smithi082 bash[79733]: cluster 2023-10-09T04:33:22.785842+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v537: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.6 MiB/s wr, 42 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:24.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.208 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:24.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:24.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:24.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:24.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:24.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:24.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:24.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:33:24.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:24.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:24.859 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:24.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 87335 87319 72.7468 0 - 0.0110854 2023-10-09T04:33:24.962 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:24.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:25.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:25.125 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:25.226 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:25.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:25.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:25.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:25.383 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:25.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:25.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:25.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:25.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:25.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:25.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:25 smithi082 bash[77063]: cluster 2023-10-09T04:33:24.787059+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v538: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:25.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:25 smithi082 bash[79733]: cluster 2023-10-09T04:33:24.787059+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v538: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:25.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:25.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:25.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 87335 87319 71.7898 0 - 0.0110854 2023-10-09T04:33:25.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:25.898 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:25 smithi153 bash[59565]: cluster 2023-10-09T04:33:24.787059+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v538: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:25.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:25.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:25.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:26.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:26.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:26.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:26.153 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:26.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:26.454 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:26.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:26.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:26.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:26.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:26.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 16 87335 87319 70.8576 0 - 0.0110854 2023-10-09T04:33:26.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:26.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:26.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:27.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:27.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:27.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:27.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:27.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:27.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:27.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:27.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:27.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:27.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:27.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 87335 87319 69.9492 0 - 0.0110854 2023-10-09T04:33:27.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.910 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:27.913 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:27 smithi082 bash[77063]: cluster 2023-10-09T04:33:26.788456+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v539: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:27.913 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:27 smithi082 bash[79733]: cluster 2023-10-09T04:33:26.788456+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v539: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:27.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:27.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:27.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:28.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:28.057 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:28.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:27 smithi153 bash[59565]: cluster 2023-10-09T04:33:26.788456+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v539: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:28.186 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:28.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:28.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:28.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:28.452 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:28.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:28.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:28.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:28.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:28.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:28.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:28.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 87335 87319 69.0639 0 - 0.0110854 2023-10-09T04:33:28.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:28.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:28.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:29.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:29.191 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:29.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:29.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:29.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:29.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:29.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:29.480 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:29.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:29.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:29.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:29.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:29.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.871 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:33:29.871185+0000 min lat: 0.00116931 max lat: 1.03787 avg lat: 0.0110854 2023-10-09T04:33:29.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:33:29.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 87335 87319 68.2007 0 - 0.0110854 2023-10-09T04:33:29.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:29.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:29.928 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:29] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.33.4" 2023-10-09T04:33:29.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:30.110 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:30.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:29 smithi153 bash[59565]: cluster 2023-10-09T04:33:28.789437+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v540: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:29 smithi153 bash[59565]: audit 2023-10-09T04:33:29.272600+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:29 smithi153 bash[59565]: audit 2023-10-09T04:33:29.273785+0000 mon.a (mon.0) 845 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:30.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:30.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[77063]: cluster 2023-10-09T04:33:28.789437+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v540: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[77063]: audit 2023-10-09T04:33:29.272600+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[77063]: audit 2023-10-09T04:33:29.273785+0000 mon.a (mon.0) 845 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[79733]: cluster 2023-10-09T04:33:28.789437+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v540: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[79733]: audit 2023-10-09T04:33:29.272600+0000 mon.a (mon.0) 844 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:30.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:29 smithi082 bash[79733]: audit 2023-10-09T04:33:29.273785+0000 mon.a (mon.0) 845 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:30.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:30.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:30.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:30.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:30.622 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:30.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:30.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:30.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 87335 87319 67.3588 0 - 0.0110854 2023-10-09T04:33:30.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:30.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:30.926 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:30.927 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:31.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:31.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:31.143 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:31.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:31.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:31.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:31.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:31.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:31.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:31.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:31.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:31.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:31.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:31.693 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:31.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:31.830 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:31.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 87335 87319 66.5374 0 - 0.0110854 2023-10-09T04:33:31.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:31.970 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:31.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:31.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:32.007 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:31 smithi082 bash[77063]: cluster 2023-10-09T04:33:30.790496+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v541: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:32.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:31 smithi082 bash[79733]: cluster 2023-10-09T04:33:30.790496+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v541: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:32.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:32.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:31 smithi153 bash[59565]: cluster 2023-10-09T04:33:30.790496+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v541: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:32.108 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:32.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:32.247 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:32.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:32.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:32.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:32.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:32.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:32.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:32.614 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:32.639 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:32.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:32.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:32.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:32.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 16 87335 87319 65.7359 0 - 0.0110854 2023-10-09T04:33:32.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:32.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:32.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:33.028 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:33.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:33.132 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:33.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:33.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:33.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:33.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:33.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:33.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:33.552 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:33.655 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:33.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:33.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:33.796 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:33.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:33.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:33.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 87335 87319 64.9534 0 - 0.0110854 2023-10-09T04:33:33.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:33.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:33.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:34.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:34.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:33 smithi153 bash[59565]: cluster 2023-10-09T04:33:32.791225+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v542: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:34.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:34.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:33 smithi082 bash[77063]: cluster 2023-10-09T04:33:32.791225+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v542: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:34.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:33 smithi082 bash[79733]: cluster 2023-10-09T04:33:32.791225+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v542: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:34.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:34.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:34.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:34.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.634 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:34.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:33:34.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:34.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:34.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:34.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 87335 87319 64.1893 0 - 0.0110854 2023-10-09T04:33:34.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:34.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:35.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:35.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:35.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:35.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:35.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:35.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.397 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:35.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:35.616 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:33:35.618 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:33:35.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:35.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:35.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:35.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:35.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 87335 87319 63.443 0 - 0.0110854 2023-10-09T04:33:35.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:35.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:35.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:35.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:35 smithi153 bash[59565]: cluster 2023-10-09T04:33:34.791972+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v543: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:35.932 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:35 smithi082 bash[77063]: cluster 2023-10-09T04:33:34.791972+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v543: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:35.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:35 smithi082 bash[79733]: cluster 2023-10-09T04:33:34.791972+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v543: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:36.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:36.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:36.168 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:36.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:36.302 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:36.406 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:36.433 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:36.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:36.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:36.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.680 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:36 smithi082 bash[77063]: audit 2023-10-09T04:33:35.595107+0000 mgr.y (mgr.34107) 580 : audit 0 from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.680 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:36 smithi082 bash[79733]: audit 2023-10-09T04:33:35.595107+0000 mgr.y (mgr.34107) 580 : audit 0 from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:36.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:36.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 87335 87319 62.7139 0 - 0.0110854 2023-10-09T04:33:36.924 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:36 smithi153 bash[59565]: audit 2023-10-09T04:33:35.595107+0000 mgr.y (mgr.34107) 580 : audit 0 from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:36.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:37.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:37.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:37.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:37.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:37.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:37.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:37.335 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:37.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:37.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:37.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:37.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:37.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:37.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:37.836 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:37 smithi082 bash[77063]: cluster 2023-10-09T04:33:36.793141+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v544: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:37.837 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:37 smithi082 bash[79733]: cluster 2023-10-09T04:33:36.793141+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v544: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:37.850 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:37.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:37.872 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 87335 87319 62.0013 0 - 0.0110854 2023-10-09T04:33:37.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:37 smithi153 bash[59565]: cluster 2023-10-09T04:33:36.793141+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v544: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:37.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:37.974 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:37.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:37.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:38.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:38.108 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:38.154 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":543,"stamp":"2023-10-09T04:33:34.791411+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5727393715,"num_objects":87348,"num_object_clones":0,"num_object_copies":262044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87348,"num_whiteouts":0,"num_read":6301,"num_read_kb":12735,"num_write":88426,"num_write_kb":5603639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88490,"ondisk_log_size":88490,"up":205,"acting":206,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20494552,"kb_used_data":18513240,"kb_used_omap":34,"kb_used_meta":1981213,"kb_avail":729466664,"statfs":{"total":767960285184,"available":746973863936,"internally_reserved":0,"allocated":18957557760,"data_stored":18952908824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35115,"internal_metadata":2028762837},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006578"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":123,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.747430+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:33:10.747430+0000","last_peered":"2023-10-09T04:33:10.747430+0000","last_clean":"2023-10-09T04:33:10.747430+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:33:10.747430+0000","last_undegraded":"2023-10-09T04:33:10.747430+0000","last_fullsized":"2023-10-09T04:33:10.747430+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":113,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.746943+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:33:10.746943+0000","last_peered":"2023-10-09T04:33:10.746943+0000","last_clean":"2023-10-09T04:33:10.746943+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:33:10.746943+0000","last_undegraded":"2023-10-09T04:33:10.746943+0000","last_fullsized":"2023-10-09T04:33:10.746943+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.758703+0000","last_change":"2023-10-09T04:33:10.758303+0000","last_active":"2023-10-09T04:33:10.758703+0000","last_peered":"2023-10-09T04:33:10.758703+0000","last_clean":"2023-10-09T04:33:01.725795+0000","last_became_active":"2023-10-09T04:33:10.758303+0000","last_became_peered":"2023-10-09T04:33:10.758303+0000","last_unstale":"2023-10-09T04:33:10.758703+0000","last_undegraded":"2023-10-09T04:33:10.758703+0000","last_fullsized":"2023-10-09T04:33:10.743169+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:52.429884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":94,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.748144+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:33:10.748144+0000","last_peered":"2023-10-09T04:33:10.748144+0000","last_clean":"2023-10-09T04:33:10.748144+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:33:10.748144+0000","last_undegraded":"2023-10-09T04:33:10.748144+0000","last_fullsized":"2023-10-09T04:33:10.748144+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.754871+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:33:10.754871+0000","last_peered":"2023-10-09T04:33:10.754871+0000","last_clean":"2023-10-09T04:33:10.754871+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:33:10.754871+0000","last_undegraded":"2023-10-09T04:33:10.754871+0000","last_fullsized":"2023-10-09T04:33:10.754871+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":140,"state":"undersized+peered","last_fresh":"2023-10-09T04:33:14.310090+0000","last_change":"2023-10-09T04:33:10.744864+0000","last_active":"2023-10-09T04:28:26.594724+0000","last_peered":"2023-10-09T04:33:14.310090+0000","last_clean":"2023-10-09T04:28:19.577947+0000","last_became_active":"2023-10-09T04:22:50.702392+0000","last_became_peered":"2023-10-09T04:33:10.744864+0000","last_unstale":"2023-10-09T04:33:14.310090+0000","last_undegraded":"2023-10-09T04:33:14.310090+0000","last_fullsized":"2023-10-09T04:33:10.743160+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:26.402983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":99,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.748080+0000","last_change":"2023-10-09T04:28:27.599874+0000","last_active":"2023-10-09T04:33:10.748080+0000","last_peered":"2023-10-09T04:33:10.748080+0000","last_clean":"2023-10-09T04:28:19.571527+0000","last_became_active":"2023-10-09T04:28:27.599874+0000","last_became_peered":"2023-10-09T04:28:27.599874+0000","last_unstale":"2023-10-09T04:33:10.748080+0000","last_undegraded":"2023-10-09T04:33:10.748080+0000","last_fullsized":"2023-10-09T04:28:27.597863+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:45.767094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":107,"reported_epoch":141,"state":"active+undersized","last_fresh":"2023-10-09T04:33:22.437771+0000","last_change":"2023-10-09T04:33:10.766893+0000","last_active":"2023-10-09T04:33:22.437771+0000","last_peered":"2023-10-09T04:33:22.437771+0000","last_clean":"2023-10-09T04:33:01.725589+0000","last_became_active":"2023-10-09T04:33:10.766893+0000","last_became_peered":"2023-10-09T04:33:10.766893+0000","last_unstale":"2023-10-09T04:33:22.437771+0000","last_undegraded":"2023-10-09T04:33:22.437771+0000","last_fullsized":"2023-10-09T04:33:10.742636+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:51.201435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":87,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.753971+0000","last_change":"2023-10-09T04:28:27.608355+0000","last_active":"2023-10-09T04:33:10.753971+0000","last_peered":"2023-10-09T04:33:10.753971+0000","last_clean":"2023-10-09T04:22:23.598572+0000","last_became_active":"2023-10-09T04:28:27.608355+0000","last_became_peered":"2023-10-09T04:28:27.608355+0000","last_unstale":"2023-10-09T04:33:10.753971+0000","last_undegraded":"2023-10-09T04:33:10.753971+0000","last_fullsized":"2023-10-09T04:28:27.604613+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.756513+0000","last_change":"2023-10-09T04:33:10.756085+0000","last_active":"2023-10-09T04:33:10.756513+0000","last_peered":"2023-10-09T04:33:10.756513+0000","last_clean":"2023-10-09T04:33:01.723744+0000","last_became_active":"2023-10-09T04:33:10.756085+0000","last_became_peered":"2023-10-09T04:33:10.756085+0000","last_unstale":"2023-10-09T04:33:10.756513+0000","last_undegraded":"2023-10-09T04:33:10.756513+0000","last_fullsized":"2023-10-09T04:33:10.755044+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":110,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.754759+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:33:10.754759+0000","last_peered":"2023-10-09T04:33:10.754759+0000","last_clean":"2023-10-09T04:33:10.754759+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:33:10.754759+0000","last_undegraded":"2023-10-09T04:33:10.754759+0000","last_fullsized":"2023-10-09T04:33:10.754759+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.756462+0000","last_change":"2023-10-09T04:33:10.755865+0000","last_active":"2023-10-09T04:33:10.756462+0000","last_peered":"2023-10-09T04:33:10.756462+0000","last_clean":"2023-10-09T04:33:01.726136+0000","last_became_active":"2023-10-09T04:33:10.755865+0000","last_became_peered":"2023-10-09T04:33:10.755865+0000","last_unstale":"2023-10-09T04:33:10.756462+0000","last_undegraded":"2023-10-09T04:33:10.756462+0000","last_fullsized":"2023-10-09T04:33:10.743735+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.889176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":120,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.756799+0000","last_change":"2023-10-09T04:33:10.756416+0000","last_active":"2023-10-09T04:33:10.756799+0000","last_peered":"2023-10-09T04:33:10.756799+0000","last_clean":"2023-10-09T04:33:01.724881+0000","last_became_active":"2023-10-09T04:33:10.756416+0000","last_became_peered":"2023-10-09T04:33:10.756416+0000","last_unstale":"2023-10-09T04:33:10.756799+0000","last_undegraded":"2023-10-09T04:33:10.746981+0000","last_fullsized":"2023-10-09T04:33:10.746831+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":112,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.750679+0000","last_change":"2023-10-09T04:33:10.750404+0000","last_active":"2023-10-09T04:33:10.750679+0000","last_peered":"2023-10-09T04:33:10.750679+0000","last_clean":"2023-10-09T04:33:01.723468+0000","last_became_active":"2023-10-09T04:33:10.750404+0000","last_became_peered":"2023-10-09T04:33:10.750404+0000","last_unstale":"2023-10-09T04:33:10.750679+0000","last_undegraded":"2023-10-09T04:33:10.750679+0000","last_fullsized":"2023-10-09T04:33:10.746070+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.755191+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:33:10.755191+0000","last_peered":"2023-10-09T04:33:10.755191+0000","last_clean":"2023-10-09T04:33:10.755191+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:33:10.755191+0000","last_undegraded":"2023-10-09T04:33:10.755191+0000","last_fullsized":"2023-10-09T04:33:10.755191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T04:33:22.437737+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:33:22.437737+0000","last_peered":"2023-10-09T04:33:22.437737+0000","last_clean":"2023-10-09T04:33:22.437737+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:33:22.437737+0000","last_undegraded":"2023-10-09T04:33:22.437737+0000","last_fullsized":"2023-10-09T04:33:22.437737+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.748315+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:33:10.748315+0000","last_peered":"2023-10-09T04:33:10.748315+0000","last_clean":"2023-10-09T04:33:10.748315+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:33:10.748315+0000","last_undegraded":"2023-10-09T04:33:10.748315+0000","last_fullsized":"2023-10-09T04:33:10.748315+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.754850+0000","last_change":"2023-10-09T04:28:27.609484+0000","last_active":"2023-10-09T04:33:10.754850+0000","last_peered":"2023-10-09T04:33:10.754850+0000","last_clean":"2023-10-09T04:23:18.758860+0000","last_became_active":"2023-10-09T04:28:27.609484+0000","last_became_peered":"2023-10-09T04:28:27.609484+0000","last_unstale":"2023-10-09T04:33:10.754850+0000","last_undegraded":"2023-10-09T04:33:10.754850+0000","last_fullsized":"2023-10-09T04:28:27.606325+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.755859+0000","last_change":"2023-10-09T04:33:10.755627+0000","last_active":"2023-10-09T04:33:10.755859+0000","last_peered":"2023-10-09T04:33:10.755859+0000","last_clean":"2023-10-09T04:33:01.723836+0000","last_became_active":"2023-10-09T04:33:10.755627+0000","last_became_peered":"2023-10-09T04:33:10.755627+0000","last_unstale":"2023-10-09T04:33:10.755859+0000","last_undegraded":"2023-10-09T04:33:10.755859+0000","last_fullsized":"2023-10-09T04:33:10.753151+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":93,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.748101+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:33:10.748101+0000","last_peered":"2023-10-09T04:33:10.748101+0000","last_clean":"2023-10-09T04:33:10.748101+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:33:10.748101+0000","last_undegraded":"2023-10-09T04:33:10.748101+0000","last_fullsized":"2023-10-09T04:33:10.748101+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:12.701560+0000","last_change":"2023-10-09T04:33:10.753825+0000","last_active":"2023-10-09T04:33:09.747157+0000","last_peered":"2023-10-09T04:33:12.701560+0000","last_clean":"2023-10-09T04:11:28.922589+0000","last_became_active":"2023-10-09T04:11:26.215822+0000","last_became_peered":"2023-10-09T04:33:10.753825+0000","last_unstale":"2023-10-09T04:33:12.701560+0000","last_undegraded":"2023-10-09T04:33:10.747853+0000","last_fullsized":"2023-10-09T04:33:10.747749+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:29:41.410029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":99,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.753790+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:33:10.753790+0000","last_peered":"2023-10-09T04:33:10.753790+0000","last_clean":"2023-10-09T04:33:10.753790+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:33:10.753790+0000","last_undegraded":"2023-10-09T04:33:10.753790+0000","last_fullsized":"2023-10-09T04:33:10.753790+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"138'5575","reported_seq":5591,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:12.701723+0000","last_change":"2023-10-09T04:33:10.749164+0000","last_active":"2023-10-09T04:33:09.742930+0000","last_peered":"2023-10-09T04:33:12.701723+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.741268+0000","last_became_peered":"2023-10-09T04:33:10.749164+0000","last_unstale":"2023-10-09T04:33:12.701723+0000","last_undegraded":"2023-10-09T04:33:10.747881+0000","last_fullsized":"2023-10-09T04:33:10.747788+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:18.909356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365363200,"num_objects":5575,"num_object_clones":0,"num_object_copies":16725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5575,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":5575}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":108,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.754217+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:33:10.754217+0000","last_peered":"2023-10-09T04:33:10.754217+0000","last_clean":"2023-10-09T04:33:10.754217+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:33:10.754217+0000","last_undegraded":"2023-10-09T04:33:10.754217+0000","last_fullsized":"2023-10-09T04:33:10.754217+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":140,"state":"undersized+peered","last_fresh":"2023-10-09T04:33:10.754882+0000","last_change":"2023-10-09T04:33:10.754620+0000","last_active":"2023-10-09T04:33:09.750956+0000","last_peered":"2023-10-09T04:33:10.754882+0000","last_clean":"2023-10-09T04:11:26.190073+0000","last_became_active":"2023-10-09T04:28:27.610146+0000","last_became_peered":"2023-10-09T04:33:10.754620+0000","last_unstale":"2023-10-09T04:33:10.754882+0000","last_undegraded":"2023-10-09T04:33:10.754882+0000","last_fullsized":"2023-10-09T04:33:10.753501+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"140'5361","reported_seq":5378,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.849686+0000","last_change":"2023-10-09T04:31:52.743124+0000","last_active":"2023-10-09T04:33:10.849686+0000","last_peered":"2023-10-09T04:33:10.849686+0000","last_clean":"2023-10-09T04:33:10.849686+0000","last_became_active":"2023-10-09T04:31:52.742658+0000","last_became_peered":"2023-10-09T04:31:52.742658+0000","last_unstale":"2023-10-09T04:33:10.849686+0000","last_undegraded":"2023-10-09T04:33:10.849686+0000","last_fullsized":"2023-10-09T04:33:10.849686+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351338496,"num_objects":5361,"num_object_clones":0,"num_object_copies":16083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5361,"num_write_kb":343104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":122,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.753761+0000","last_change":"2023-10-09T04:29:12.722710+0000","last_active":"2023-10-09T04:33:10.753761+0000","last_peered":"2023-10-09T04:33:10.753761+0000","last_clean":"2023-10-09T04:33:10.753761+0000","last_became_active":"2023-10-09T04:29:12.635166+0000","last_became_peered":"2023-10-09T04:29:12.635166+0000","last_unstale":"2023-10-09T04:33:10.753761+0000","last_undegraded":"2023-10-09T04:33:10.753761+0000","last_fullsized":"2023-10-09T04:33:10.753761+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":121,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.747680+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:33:10.747680+0000","last_peered":"2023-10-09T04:33:10.747680+0000","last_clean":"2023-10-09T04:33:10.747680+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:33:10.747680+0000","last_undegraded":"2023-10-09T04:33:10.747680+0000","last_fullsized":"2023-10-09T04:33:10.747680+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"140'5549","reported_seq":5566,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.875070+0000","last_change":"2023-10-09T04:31:52.742863+0000","last_active":"2023-10-09T04:33:10.875070+0000","last_peered":"2023-10-09T04:33:10.875070+0000","last_clean":"2023-10-09T04:33:10.875070+0000","last_became_active":"2023-10-09T04:31:52.742528+0000","last_became_peered":"2023-10-09T04:31:52.742528+0000","last_unstale":"2023-10-09T04:33:10.875070+0000","last_undegraded":"2023-10-09T04:33:10.875070+0000","last_fullsized":"2023-10-09T04:33:10.875070+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5549,"log_dups_size":0,"ondisk_log_size":5549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363659264,"num_objects":5549,"num_object_clones":0,"num_object_copies":16647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5549,"num_write_kb":355136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":104,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:10.755887+0000","last_change":"2023-10-09T04:33:10.755611+0000","last_active":"2023-10-09T04:33:09.756369+0000","last_peered":"2023-10-09T04:33:10.755887+0000","last_clean":"2023-10-09T04:28:19.573266+0000","last_became_active":"2023-10-09T04:28:27.610412+0000","last_became_peered":"2023-10-09T04:33:10.755611+0000","last_unstale":"2023-10-09T04:33:10.755887+0000","last_undegraded":"2023-10-09T04:33:10.754524+0000","last_fullsized":"2023-10-09T04:33:10.754417+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:14.309805+0000","last_change":"2023-10-09T04:33:10.755945+0000","last_active":"2023-10-09T04:33:14.309805+0000","last_peered":"2023-10-09T04:33:14.309805+0000","last_clean":"2023-10-09T04:22:23.601355+0000","last_became_active":"2023-10-09T04:33:10.755945+0000","last_became_peered":"2023-10-09T04:33:10.755945+0000","last_unstale":"2023-10-09T04:33:14.309805+0000","last_undegraded":"2023-10-09T04:33:14.309805+0000","last_fullsized":"2023-10-09T04:33:10.742942+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:55.708485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"140'5507","reported_seq":5525,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.874369+0000","last_change":"2023-10-09T04:31:52.743807+0000","last_active":"2023-10-09T04:33:10.874369+0000","last_peered":"2023-10-09T04:33:10.874369+0000","last_clean":"2023-10-09T04:33:10.874369+0000","last_became_active":"2023-10-09T04:31:52.743450+0000","last_became_peered":"2023-10-09T04:31:52.743450+0000","last_unstale":"2023-10-09T04:33:10.874369+0000","last_undegraded":"2023-10-09T04:33:10.874369+0000","last_fullsized":"2023-10-09T04:33:10.874369+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:52.615776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360906752,"num_objects":5507,"num_object_clones":0,"num_object_copies":16521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5507,"num_write_kb":352448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":80,"reported_epoch":140,"state":"undersized+peered","last_fresh":"2023-10-09T04:33:14.021359+0000","last_change":"2023-10-09T04:33:10.761324+0000","last_active":"2023-10-09T04:23:59.793140+0000","last_peered":"2023-10-09T04:33:14.021359+0000","last_clean":"2023-10-09T04:23:18.758933+0000","last_became_active":"2023-10-09T04:23:48.430491+0000","last_became_peered":"2023-10-09T04:33:10.761324+0000","last_unstale":"2023-10-09T04:33:14.021359+0000","last_undegraded":"2023-10-09T04:33:14.021359+0000","last_fullsized":"2023-10-09T04:33:10.749724+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:30.270894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.747447+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:33:10.747447+0000","last_peered":"2023-10-09T04:33:10.747447+0000","last_clean":"2023-10-09T04:33:10.747447+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:33:10.747447+0000","last_undegraded":"2023-10-09T04:33:10.747447+0000","last_fullsized":"2023-10-09T04:33:10.747447+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"140'5462","reported_seq":5479,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.863209+0000","last_change":"2023-10-09T04:31:52.743032+0000","last_active":"2023-10-09T04:33:10.863209+0000","last_peered":"2023-10-09T04:33:10.863209+0000","last_clean":"2023-10-09T04:33:10.863209+0000","last_became_active":"2023-10-09T04:31:52.741647+0000","last_became_peered":"2023-10-09T04:31:52.741647+0000","last_unstale":"2023-10-09T04:33:10.863209+0000","last_undegraded":"2023-10-09T04:33:10.863209+0000","last_fullsized":"2023-10-09T04:33:10.863209+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5462,"log_dups_size":0,"ondisk_log_size":5462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357957632,"num_objects":5462,"num_object_clones":0,"num_object_copies":16386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5462,"num_write_kb":349568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":57,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:22.703667+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:33:22.703667+0000","last_peered":"2023-10-09T04:33:22.703667+0000","last_clean":"2023-10-09T04:33:22.703667+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:33:22.703667+0000","last_undegraded":"2023-10-09T04:33:22.703667+0000","last_fullsized":"2023-10-09T04:33:22.703667+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.761493+0000","last_change":"2023-10-09T04:33:10.761212+0000","last_active":"2023-10-09T04:33:10.761493+0000","last_peered":"2023-10-09T04:33:10.761493+0000","last_clean":"2023-10-09T04:33:01.724876+0000","last_became_active":"2023-10-09T04:33:10.761212+0000","last_became_peered":"2023-10-09T04:33:10.761212+0000","last_unstale":"2023-10-09T04:33:10.761493+0000","last_undegraded":"2023-10-09T04:33:10.761493+0000","last_fullsized":"2023-10-09T04:33:10.747223+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"140'5436","reported_seq":5454,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.868711+0000","last_change":"2023-10-09T04:31:52.739726+0000","last_active":"2023-10-09T04:33:10.868711+0000","last_peered":"2023-10-09T04:33:10.868711+0000","last_clean":"2023-10-09T04:33:10.868711+0000","last_became_active":"2023-10-09T04:31:52.739407+0000","last_became_peered":"2023-10-09T04:31:52.739407+0000","last_unstale":"2023-10-09T04:33:10.868711+0000","last_undegraded":"2023-10-09T04:33:10.868711+0000","last_fullsized":"2023-10-09T04:33:10.868711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5436,"log_dups_size":0,"ondisk_log_size":5436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:47.453978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356253696,"num_objects":5436,"num_object_clones":0,"num_object_copies":16308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5436,"num_write_kb":347904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.750957+0000","last_change":"2023-10-09T04:33:10.750495+0000","last_active":"2023-10-09T04:33:10.750957+0000","last_peered":"2023-10-09T04:33:10.750957+0000","last_clean":"2023-10-09T04:22:23.601526+0000","last_became_active":"2023-10-09T04:33:10.750495+0000","last_became_peered":"2023-10-09T04:33:10.750495+0000","last_unstale":"2023-10-09T04:33:10.750957+0000","last_undegraded":"2023-10-09T04:33:10.750957+0000","last_fullsized":"2023-10-09T04:33:10.746253+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.753684+0000","last_change":"2023-10-09T04:33:10.753468+0000","last_active":"2023-10-09T04:33:10.753684+0000","last_peered":"2023-10-09T04:33:10.753684+0000","last_clean":"2023-10-09T04:33:01.733237+0000","last_became_active":"2023-10-09T04:33:10.753468+0000","last_became_peered":"2023-10-09T04:33:10.753468+0000","last_unstale":"2023-10-09T04:33:10.753684+0000","last_undegraded":"2023-10-09T04:33:10.753684+0000","last_fullsized":"2023-10-09T04:33:10.745724+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:27.829687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"138'5480","reported_seq":5498,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:14.021319+0000","last_change":"2023-10-09T04:33:10.757608+0000","last_active":"2023-10-09T04:33:09.753599+0000","last_peered":"2023-10-09T04:33:14.021319+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.739928+0000","last_became_peered":"2023-10-09T04:33:10.757608+0000","last_unstale":"2023-10-09T04:33:14.021319+0000","last_undegraded":"2023-10-09T04:33:10.749194+0000","last_fullsized":"2023-10-09T04:33:10.749125+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:16.010699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359137280,"num_objects":5480,"num_object_clones":0,"num_object_copies":16440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5480,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[{"shards":"3","objects":5480}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.746402+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:33:10.746402+0000","last_peered":"2023-10-09T04:33:10.746402+0000","last_clean":"2023-10-09T04:33:10.746402+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:33:10.746402+0000","last_undegraded":"2023-10-09T04:33:10.746402+0000","last_fullsized":"2023-10-09T04:33:10.746402+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":112,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.751628+0000","last_change":"2023-10-09T04:33:10.751275+0000","last_active":"2023-10-09T04:33:10.751628+0000","last_peered":"2023-10-09T04:33:10.751628+0000","last_clean":"2023-10-09T04:33:01.723420+0000","last_became_active":"2023-10-09T04:33:10.751275+0000","last_became_peered":"2023-10-09T04:33:10.751275+0000","last_unstale":"2023-10-09T04:33:10.751628+0000","last_undegraded":"2023-10-09T04:33:10.751628+0000","last_fullsized":"2023-10-09T04:33:10.746635+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"140'5409","reported_seq":5424,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.880076+0000","last_change":"2023-10-09T04:32:09.879049+0000","last_active":"2023-10-09T04:33:10.880076+0000","last_peered":"2023-10-09T04:33:10.880076+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.733735+0000","last_became_peered":"2023-10-09T04:31:52.733735+0000","last_unstale":"2023-10-09T04:33:10.880076+0000","last_undegraded":"2023-10-09T04:32:09.869011+0000","last_fullsized":"2023-10-09T04:31:52.730251+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:34.729548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354484224,"num_objects":5409,"num_object_clones":0,"num_object_copies":16227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5409,"num_write_kb":346176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":5409}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.309757+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:33:14.309757+0000","last_peered":"2023-10-09T04:33:14.309757+0000","last_clean":"2023-10-09T04:33:14.309757+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:33:14.309757+0000","last_undegraded":"2023-10-09T04:33:14.309757+0000","last_fullsized":"2023-10-09T04:33:14.309757+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":88,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.743082+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:33:10.743082+0000","last_peered":"2023-10-09T04:33:10.743082+0000","last_clean":"2023-10-09T04:33:10.743082+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:33:10.743082+0000","last_undegraded":"2023-10-09T04:33:10.743082+0000","last_fullsized":"2023-10-09T04:33:10.743082+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"140'5573","reported_seq":5591,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.863370+0000","last_change":"2023-10-09T04:31:52.736550+0000","last_active":"2023-10-09T04:33:10.863370+0000","last_peered":"2023-10-09T04:33:10.863370+0000","last_clean":"2023-10-09T04:33:10.863370+0000","last_became_active":"2023-10-09T04:31:52.736098+0000","last_became_peered":"2023-10-09T04:31:52.736098+0000","last_unstale":"2023-10-09T04:33:10.863370+0000","last_undegraded":"2023-10-09T04:33:10.863370+0000","last_fullsized":"2023-10-09T04:33:10.863370+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5573,"log_dups_size":0,"ondisk_log_size":5573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:41.783315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5573,"num_write_kb":356672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":514,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.310009+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:33:14.310009+0000","last_peered":"2023-10-09T04:33:14.310009+0000","last_clean":"2023-10-09T04:33:14.310009+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:33:14.310009+0000","last_undegraded":"2023-10-09T04:33:14.310009+0000","last_fullsized":"2023-10-09T04:33:14.310009+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":113,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.746466+0000","last_change":"2023-10-09T04:28:51.527252+0000","last_active":"2023-10-09T04:33:10.746466+0000","last_peered":"2023-10-09T04:33:10.746466+0000","last_clean":"2023-10-09T04:23:18.768463+0000","last_became_active":"2023-10-09T04:28:51.527252+0000","last_became_peered":"2023-10-09T04:28:51.527252+0000","last_unstale":"2023-10-09T04:33:10.746466+0000","last_undegraded":"2023-10-09T04:33:10.746466+0000","last_fullsized":"2023-10-09T04:28:51.524832+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:29.101177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:14.309847+0000","last_change":"2023-10-09T04:28:27.600497+0000","last_active":"2023-10-09T04:33:14.309847+0000","last_peered":"2023-10-09T04:33:14.309847+0000","last_clean":"2023-10-09T04:28:19.561863+0000","last_became_active":"2023-10-09T04:28:27.600497+0000","last_became_peered":"2023-10-09T04:28:27.600497+0000","last_unstale":"2023-10-09T04:33:14.309847+0000","last_undegraded":"2023-10-09T04:33:14.309847+0000","last_fullsized":"2023-10-09T04:28:27.591503+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:26.054422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"140'5326","reported_seq":5343,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.849897+0000","last_change":"2023-10-09T04:31:52.744989+0000","last_active":"2023-10-09T04:33:10.849897+0000","last_peered":"2023-10-09T04:33:10.849897+0000","last_clean":"2023-10-09T04:33:10.849897+0000","last_became_active":"2023-10-09T04:31:52.744622+0000","last_became_peered":"2023-10-09T04:31:52.744622+0000","last_unstale":"2023-10-09T04:33:10.849897+0000","last_undegraded":"2023-10-09T04:33:10.849897+0000","last_fullsized":"2023-10-09T04:33:10.849897+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349044736,"num_objects":5326,"num_object_clones":0,"num_object_copies":15978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":340864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":607,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:10.745169+0000","last_change":"2023-10-09T04:33:10.744593+0000","last_active":"2023-10-09T04:33:09.757685+0000","last_peered":"2023-10-09T04:33:10.745169+0000","last_clean":"2023-10-09T04:13:52.717465+0000","last_became_active":"2023-10-09T04:28:27.605360+0000","last_became_peered":"2023-10-09T04:33:10.744593+0000","last_unstale":"2023-10-09T04:33:10.745169+0000","last_undegraded":"2023-10-09T04:33:10.743325+0000","last_fullsized":"2023-10-09T04:33:10.743251+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:18.699126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":113,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.749261+0000","last_change":"2023-10-09T04:33:10.748842+0000","last_active":"2023-10-09T04:33:10.749261+0000","last_peered":"2023-10-09T04:33:10.749261+0000","last_clean":"2023-10-09T04:33:01.723379+0000","last_became_active":"2023-10-09T04:33:10.748842+0000","last_became_peered":"2023-10-09T04:33:10.748842+0000","last_unstale":"2023-10-09T04:33:10.749261+0000","last_undegraded":"2023-10-09T04:33:10.749261+0000","last_fullsized":"2023-10-09T04:33:10.747336+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":104,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.309906+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:33:14.309906+0000","last_peered":"2023-10-09T04:33:14.309906+0000","last_clean":"2023-10-09T04:33:14.309906+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:33:14.309906+0000","last_undegraded":"2023-10-09T04:33:14.309906+0000","last_fullsized":"2023-10-09T04:33:14.309906+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"140'5407","reported_seq":5429,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.834940+0000","last_change":"2023-10-09T04:33:10.755792+0000","last_active":"2023-10-09T04:33:10.834940+0000","last_peered":"2023-10-09T04:33:10.834940+0000","last_clean":"2023-10-09T04:33:09.748891+0000","last_became_active":"2023-10-09T04:33:10.755792+0000","last_became_peered":"2023-10-09T04:33:10.755792+0000","last_unstale":"2023-10-09T04:33:10.834940+0000","last_undegraded":"2023-10-09T04:33:10.754090+0000","last_fullsized":"2023-10-09T04:33:10.753980+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5407,"log_dups_size":0,"ondisk_log_size":5407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354353152,"num_objects":5407,"num_object_clones":0,"num_object_copies":16221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5407,"num_write_kb":346048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":5407}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1249,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.746111+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:33:10.746111+0000","last_peered":"2023-10-09T04:33:10.746111+0000","last_clean":"2023-10-09T04:33:10.746111+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:33:10.746111+0000","last_undegraded":"2023-10-09T04:33:10.746111+0000","last_fullsized":"2023-10-09T04:33:10.746111+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.750885+0000","last_change":"2023-10-09T04:33:10.750675+0000","last_active":"2023-10-09T04:33:10.750885+0000","last_peered":"2023-10-09T04:33:10.750885+0000","last_clean":"2023-10-09T04:33:01.725404+0000","last_became_active":"2023-10-09T04:33:10.750675+0000","last_became_peered":"2023-10-09T04:33:10.750675+0000","last_unstale":"2023-10-09T04:33:10.750885+0000","last_undegraded":"2023-10-09T04:33:10.750885+0000","last_fullsized":"2023-10-09T04:33:10.742640+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:31.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.742958+0000","last_change":"2023-10-09T04:28:27.599081+0000","last_active":"2023-10-09T04:33:10.742958+0000","last_peered":"2023-10-09T04:33:10.742958+0000","last_clean":"2023-10-09T04:28:19.562734+0000","last_became_active":"2023-10-09T04:28:27.599081+0000","last_became_peered":"2023-10-09T04:28:27.599081+0000","last_unstale":"2023-10-09T04:33:10.742958+0000","last_undegraded":"2023-10-09T04:33:10.742958+0000","last_fullsized":"2023-10-09T04:28:27.592506+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:57.226039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"140'5537","reported_seq":5551,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.826503+0000","last_change":"2023-10-09T04:32:09.866092+0000","last_active":"2023-10-09T04:33:10.826503+0000","last_peered":"2023-10-09T04:33:10.826503+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.740185+0000","last_became_peered":"2023-10-09T04:31:52.740185+0000","last_unstale":"2023-10-09T04:33:10.826503+0000","last_undegraded":"2023-10-09T04:32:09.859653+0000","last_fullsized":"2023-10-09T04:31:52.735459+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362872832,"num_objects":5537,"num_object_clones":0,"num_object_copies":16611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5537,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5537,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":5537}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":578,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.745916+0000","last_change":"2023-10-09T04:28:27.614054+0000","last_active":"2023-10-09T04:33:10.745916+0000","last_peered":"2023-10-09T04:33:10.745916+0000","last_clean":"2023-10-09T04:13:52.699710+0000","last_became_active":"2023-10-09T04:28:27.614054+0000","last_became_peered":"2023-10-09T04:28:27.614054+0000","last_unstale":"2023-10-09T04:33:10.745916+0000","last_undegraded":"2023-10-09T04:28:27.594108+0000","last_fullsized":"2023-10-09T04:28:27.593983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:21.875430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":349,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.757528+0000","last_change":"2023-10-09T04:33:10.757248+0000","last_active":"2023-10-09T04:33:10.757528+0000","last_peered":"2023-10-09T04:33:10.757528+0000","last_clean":"2023-10-09T04:33:01.724174+0000","last_became_active":"2023-10-09T04:33:10.757248+0000","last_became_peered":"2023-10-09T04:33:10.757248+0000","last_unstale":"2023-10-09T04:33:10.757528+0000","last_undegraded":"2023-10-09T04:33:10.753894+0000","last_fullsized":"2023-10-09T04:33:10.753712+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":140,"state":"undersized+peered","last_fresh":"2023-10-09T04:33:14.021280+0000","last_change":"2023-10-09T04:33:10.753038+0000","last_active":"2023-10-09T04:28:26.601109+0000","last_peered":"2023-10-09T04:33:14.021280+0000","last_clean":"2023-10-09T04:28:19.566551+0000","last_became_active":"2023-10-09T04:24:00.819513+0000","last_became_peered":"2023-10-09T04:33:10.753038+0000","last_unstale":"2023-10-09T04:33:14.021280+0000","last_undegraded":"2023-10-09T04:33:14.021280+0000","last_fullsized":"2023-10-09T04:33:10.748705+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:43.146597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.758522+0000","last_change":"2023-10-09T04:33:10.758268+0000","last_active":"2023-10-09T04:33:10.758522+0000","last_peered":"2023-10-09T04:33:10.758522+0000","last_clean":"2023-10-09T04:11:26.190073+0000","last_became_active":"2023-10-09T04:33:10.758268+0000","last_became_peered":"2023-10-09T04:33:10.758268+0000","last_unstale":"2023-10-09T04:33:10.758522+0000","last_undegraded":"2023-10-09T04:33:10.758522+0000","last_fullsized":"2023-10-09T04:33:10.748423+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"140'5459","reported_seq":5474,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.893191+0000","last_change":"2023-10-09T04:32:09.866362+0000","last_active":"2023-10-09T04:33:10.893191+0000","last_peered":"2023-10-09T04:33:10.893191+0000","last_clean":"2023-10-09T04:31:51.707191+0000","last_became_active":"2023-10-09T04:31:52.731757+0000","last_became_peered":"2023-10-09T04:31:52.731757+0000","last_unstale":"2023-10-09T04:33:10.893191+0000","last_undegraded":"2023-10-09T04:32:09.854852+0000","last_fullsized":"2023-10-09T04:31:52.728894+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:57.001512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357761024,"num_objects":5459,"num_object_clones":0,"num_object_copies":16377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5459,"num_write_kb":349376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":5459}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":685,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.754443+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:33:10.754443+0000","last_peered":"2023-10-09T04:33:10.754443+0000","last_clean":"2023-10-09T04:33:10.754443+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:33:10.754443+0000","last_undegraded":"2023-10-09T04:33:10.754443+0000","last_fullsized":"2023-10-09T04:33:10.754443+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":140,"state":"active+clean+remapped","last_fresh":"2023-10-09T04:33:10.747368+0000","last_change":"2023-10-09T04:29:13.767749+0000","last_active":"2023-10-09T04:33:10.747368+0000","last_peered":"2023-10-09T04:33:10.747368+0000","last_clean":"2023-10-09T04:33:10.747368+0000","last_became_active":"2023-10-09T04:29:13.767307+0000","last_became_peered":"2023-10-09T04:29:13.767307+0000","last_unstale":"2023-10-09T04:33:10.747368+0000","last_undegraded":"2023-10-09T04:33:10.747368+0000","last_fullsized":"2023-10-09T04:33:10.747368+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:30.355543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:12.701937+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:33:12.701937+0000","last_peered":"2023-10-09T04:33:12.701937+0000","last_clean":"2023-10-09T04:33:12.701937+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:33:12.701937+0000","last_undegraded":"2023-10-09T04:33:12.701937+0000","last_fullsized":"2023-10-09T04:33:12.701937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"140'5442","reported_seq":5466,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.863811+0000","last_change":"2023-10-09T04:33:10.748262+0000","last_active":"2023-10-09T04:33:10.863811+0000","last_peered":"2023-10-09T04:33:10.863811+0000","last_clean":"2023-10-09T04:33:09.730942+0000","last_became_active":"2023-10-09T04:33:10.748262+0000","last_became_peered":"2023-10-09T04:33:10.748262+0000","last_unstale":"2023-10-09T04:33:10.863811+0000","last_undegraded":"2023-10-09T04:33:10.742934+0000","last_fullsized":"2023-10-09T04:33:10.742811+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:35.318781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5442,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":348288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":5442}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":354,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.754695+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:33:10.754695+0000","last_peered":"2023-10-09T04:33:10.754695+0000","last_clean":"2023-10-09T04:33:10.754695+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:33:10.754695+0000","last_undegraded":"2023-10-09T04:33:10.754695+0000","last_fullsized":"2023-10-09T04:33:10.754695+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:14.309925+0000","last_change":"2023-10-09T04:28:27.599163+0000","last_active":"2023-10-09T04:33:14.309925+0000","last_peered":"2023-10-09T04:33:14.309925+0000","last_clean":"2023-10-09T04:28:19.563114+0000","last_became_active":"2023-10-09T04:28:27.599163+0000","last_became_peered":"2023-10-09T04:28:27.599163+0000","last_unstale":"2023-10-09T04:33:14.309925+0000","last_undegraded":"2023-10-09T04:33:14.309925+0000","last_fullsized":"2023-10-09T04:28:27.591374+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:37.443801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":85,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.748400+0000","last_change":"2023-10-09T04:28:27.610457+0000","last_active":"2023-10-09T04:33:10.748400+0000","last_peered":"2023-10-09T04:33:10.748400+0000","last_clean":"2023-10-09T04:28:19.571509+0000","last_became_active":"2023-10-09T04:28:27.610457+0000","last_became_peered":"2023-10-09T04:28:27.610457+0000","last_unstale":"2023-10-09T04:33:10.748400+0000","last_undegraded":"2023-10-09T04:33:10.748400+0000","last_fullsized":"2023-10-09T04:28:27.598190+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:23.684166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"140'5466","reported_seq":5489,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.870833+0000","last_change":"2023-10-09T04:33:10.754551+0000","last_active":"2023-10-09T04:33:10.870833+0000","last_peered":"2023-10-09T04:33:10.870833+0000","last_clean":"2023-10-09T04:33:09.699225+0000","last_became_active":"2023-10-09T04:33:10.754551+0000","last_became_peered":"2023-10-09T04:33:10.754551+0000","last_unstale":"2023-10-09T04:33:10.870833+0000","last_undegraded":"2023-10-09T04:33:10.746580+0000","last_fullsized":"2023-10-09T04:33:10.746470+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:14.718355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358219776,"num_objects":5466,"num_object_clones":0,"num_object_copies":16398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":349824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":5466}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":55,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.753658+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:33:10.753658+0000","last_peered":"2023-10-09T04:33:10.753658+0000","last_clean":"2023-10-09T04:33:10.753658+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:33:10.753658+0000","last_undegraded":"2023-10-09T04:33:10.753658+0000","last_fullsized":"2023-10-09T04:33:10.753658+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":370,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.751086+0000","last_change":"2023-10-09T04:33:10.750699+0000","last_active":"2023-10-09T04:33:10.751086+0000","last_peered":"2023-10-09T04:33:10.751086+0000","last_clean":"2023-10-09T04:33:01.726548+0000","last_became_active":"2023-10-09T04:33:10.750699+0000","last_became_peered":"2023-10-09T04:33:10.750699+0000","last_unstale":"2023-10-09T04:33:10.751086+0000","last_undegraded":"2023-10-09T04:33:10.751086+0000","last_fullsized":"2023-10-09T04:33:10.743826+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:36.866478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.753828+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:33:10.753828+0000","last_peered":"2023-10-09T04:33:10.753828+0000","last_clean":"2023-10-09T04:33:10.753828+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:33:10.753828+0000","last_undegraded":"2023-10-09T04:33:10.753828+0000","last_fullsized":"2023-10-09T04:33:10.753828+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.3","version":"140'5331","reported_seq":5354,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.884345+0000","last_change":"2023-10-09T04:33:10.756493+0000","last_active":"2023-10-09T04:33:10.884345+0000","last_peered":"2023-10-09T04:33:10.884345+0000","last_clean":"2023-10-09T04:33:09.740636+0000","last_became_active":"2023-10-09T04:33:10.756493+0000","last_became_peered":"2023-10-09T04:33:10.756493+0000","last_unstale":"2023-10-09T04:33:10.884345+0000","last_undegraded":"2023-10-09T04:33:10.753454+0000","last_fullsized":"2023-10-09T04:33:10.753215+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":132,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:31:51.707191+0000","last_clean_scrub_stamp":"2023-10-09T04:31:51.707191+0000","objects_scrubbed":0,"log_size":5331,"log_dups_size":0,"ondisk_log_size":5331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5331,"num_write_kb":341184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":5331}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":117,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.756911+0000","last_change":"2023-10-09T04:33:10.756546+0000","last_active":"2023-10-09T04:33:10.756911+0000","last_peered":"2023-10-09T04:33:10.756911+0000","last_clean":"2023-10-09T04:13:51.187255+0000","last_became_active":"2023-10-09T04:33:10.756546+0000","last_became_peered":"2023-10-09T04:33:10.756546+0000","last_unstale":"2023-10-09T04:33:10.756911+0000","last_undegraded":"2023-10-09T04:33:10.756911+0000","last_fullsized":"2023-10-09T04:33:10.753998+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:12.701910+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:33:12.701910+0000","last_peered":"2023-10-09T04:33:12.701910+0000","last_clean":"2023-10-09T04:33:12.701910+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:33:12.701910+0000","last_undegraded":"2023-10-09T04:33:12.701910+0000","last_fullsized":"2023-10-09T04:33:12.701910+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":102,"reported_epoch":140,"state":"undersized+peered","last_fresh":"2023-10-09T04:33:10.747987+0000","last_change":"2023-10-09T04:33:10.747727+0000","last_active":"2023-10-09T04:33:09.748076+0000","last_peered":"2023-10-09T04:33:10.747987+0000","last_clean":"2023-10-09T04:28:19.571550+0000","last_became_active":"2023-10-09T04:28:27.605015+0000","last_became_peered":"2023-10-09T04:33:10.747727+0000","last_unstale":"2023-10-09T04:33:10.747987+0000","last_undegraded":"2023-10-09T04:33:10.747987+0000","last_fullsized":"2023-10-09T04:33:10.746613+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":82,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.747974+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:33:10.747974+0000","last_peered":"2023-10-09T04:33:10.747974+0000","last_clean":"2023-10-09T04:33:10.747974+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:33:10.747974+0000","last_undegraded":"2023-10-09T04:33:10.747974+0000","last_fullsized":"2023-10-09T04:33:10.747974+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:17.885990+0000","last_change":"2023-10-09T04:28:27.600435+0000","last_active":"2023-10-09T04:33:17.885990+0000","last_peered":"2023-10-09T04:33:17.885990+0000","last_clean":"2023-10-09T04:28:19.572818+0000","last_became_active":"2023-10-09T04:28:27.600435+0000","last_became_peered":"2023-10-09T04:28:27.600435+0000","last_unstale":"2023-10-09T04:33:17.885990+0000","last_undegraded":"2023-10-09T04:33:17.885990+0000","last_fullsized":"2023-10-09T04:28:27.592390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:45.015770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":46,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.746782+0000","last_change":"2023-10-09T04:28:27.600816+0000","last_active":"2023-10-09T04:33:10.746782+0000","last_peered":"2023-10-09T04:33:10.746782+0000","last_clean":"2023-10-09T04:11:28.922252+0000","last_became_active":"2023-10-09T04:28:27.600816+0000","last_became_peered":"2023-10-09T04:28:27.600816+0000","last_unstale":"2023-10-09T04:33:10.746782+0000","last_undegraded":"2023-10-09T04:28:27.597550+0000","last_fullsized":"2023-10-09T04:28:27.597399+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.745365+0000","last_change":"2023-10-09T04:29:12.632856+0000","last_active":"2023-10-09T04:33:10.745365+0000","last_peered":"2023-10-09T04:33:10.745365+0000","last_clean":"2023-10-09T04:33:10.745365+0000","last_became_active":"2023-10-09T04:29:12.632465+0000","last_became_peered":"2023-10-09T04:29:12.632465+0000","last_unstale":"2023-10-09T04:33:10.745365+0000","last_undegraded":"2023-10-09T04:33:10.745365+0000","last_fullsized":"2023-10-09T04:33:10.745365+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:26.854929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":51,"reported_epoch":140,"state":"undersized+degraded+peered","last_fresh":"2023-10-09T04:33:10.749227+0000","last_change":"2023-10-09T04:33:10.748881+0000","last_active":"2023-10-09T04:33:09.748417+0000","last_peered":"2023-10-09T04:33:10.749227+0000","last_clean":"2023-10-09T04:11:28.923128+0000","last_became_active":"2023-10-09T04:28:27.606662+0000","last_became_peered":"2023-10-09T04:33:10.748881+0000","last_unstale":"2023-10-09T04:33:10.749227+0000","last_undegraded":"2023-10-09T04:33:10.748121+0000","last_fullsized":"2023-10-09T04:33:10.748015+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.748071+0000","last_change":"2023-10-09T04:29:12.632656+0000","last_active":"2023-10-09T04:33:10.748071+0000","last_peered":"2023-10-09T04:33:10.748071+0000","last_clean":"2023-10-09T04:33:10.748071+0000","last_became_active":"2023-10-09T04:29:12.632273+0000","last_became_peered":"2023-10-09T04:29:12.632273+0000","last_unstale":"2023-10-09T04:33:10.748071+0000","last_undegraded":"2023-10-09T04:33:10.748071+0000","last_fullsized":"2023-10-09T04:33:10.748071+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:27.569523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":10,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:12.885157+0000","last_change":"2023-10-09T04:33:10.754269+0000","last_active":"2023-10-09T04:33:12.885157+0000","last_peered":"2023-10-09T04:33:12.885157+0000","last_clean":"2023-10-09T04:11:25.176558+0000","last_became_active":"2023-10-09T04:33:10.754269+0000","last_became_peered":"2023-10-09T04:33:10.754269+0000","last_unstale":"2023-10-09T04:33:12.885157+0000","last_undegraded":"2023-10-09T04:33:12.885157+0000","last_fullsized":"2023-10-09T04:33:10.747143+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:17.812071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.746019+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:33:10.746019+0000","last_peered":"2023-10-09T04:33:10.746019+0000","last_clean":"2023-10-09T04:33:10.746019+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:33:10.746019+0000","last_undegraded":"2023-10-09T04:33:10.746019+0000","last_fullsized":"2023-10-09T04:33:10.746019+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":104,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:33:10.748017+0000","last_change":"2023-10-09T04:28:27.606014+0000","last_active":"2023-10-09T04:33:10.748017+0000","last_peered":"2023-10-09T04:33:10.748017+0000","last_clean":"2023-10-09T04:28:19.571450+0000","last_became_active":"2023-10-09T04:28:27.606014+0000","last_became_peered":"2023-10-09T04:28:27.606014+0000","last_unstale":"2023-10-09T04:33:10.748017+0000","last_undegraded":"2023-10-09T04:28:27.597046+0000","last_fullsized":"2023-10-09T04:28:27.594857+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:07.466276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":112,"reported_epoch":140,"state":"active+undersized","last_fresh":"2023-10-09T04:33:10.750450+0000","last_change":"2023-10-09T04:33:10.750173+0000","last_active":"2023-10-09T04:33:10.750450+0000","last_peered":"2023-10-09T04:33:10.750450+0000","last_clean":"2023-10-09T04:33:01.722760+0000","last_became_active":"2023-10-09T04:33:10.750173+0000","last_became_peered":"2023-10-09T04:33:10.750173+0000","last_unstale":"2023-10-09T04:33:10.750450+0000","last_undegraded":"2023-10-09T04:33:10.750450+0000","last_fullsized":"2023-10-09T04:33:10.746058+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5722603520,"num_objects":87320,"num_object_clones":0,"num_object_copies":261960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60161,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87320,"num_write_kb":5588480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15264448512,"data_stored":15264448512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87320,"ondisk_log_size":87320,"up":37,"acting":37,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28699,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":72,"acting":73,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430509,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2398428,"kb_used_data":2084820,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91346724,"statfs":{"total":95995035648,"available":93539045376,"internally_reserved":0,"allocated":2134855680,"data_stored":2134329208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":3.6070000000000002,"15min":1.7989999999999999},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.2599999999999998,"5min":250.65700000000001,"15min":250.65700000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.2410000000000001,"5min":4.375,"15min":2.0840000000000001},"min":{"1min":0.32400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8529999999999998,"5min":251.214,"15min":251.214},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.107,"15min":0.98099999999999998},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.4970000000000001,"5min":7.282,"15min":7.282},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.6399999999999999,"15min":1.1970000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.5049999999999999,"5min":11.988,"15min":11.988},"last":0.749}]},{"osd":2,"last update":"Mon Oct 9 04:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":2.5710000000000002,"15min":1.4830000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.9989999999999997,"5min":106.456,"15min":106.456},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.629,"15min":1.544},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.9630000000000001,"5min":106.708,"15min":106.708},"last":0.78200000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.218,"15min":1.1020000000000001},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":7.0030000000000001,"5min":7.0030000000000001,"15min":7.0030000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.655,"5min":1.228,"15min":1.1120000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.0250000000000004,"5min":7.3399999999999999,"15min":7.3399999999999999},"last":0.86199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:33:20 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.319,"15min":1.1259999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.1369999999999996,"5min":6.5780000000000003,"15min":6.5780000000000003},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.0489999999999999,"15min":1.0329999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.3769999999999998,"5min":5.431,"15min":5.431},"last":0.88700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.0600000000000001,"15min":1.0089999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.0209999999999999,"5min":4.0209999999999999,"15min":4.0209999999999999},"last":5.4470000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.2,"15min":1.0800000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":1.9219999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:32:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.3959999999999999,"15min":1.1759999999999999},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2170000000000001,"5min":6.5460000000000003,"15min":6.5460000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1020000000000001,"15min":1.0720000000000001},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.601,"5min":5.4050000000000002,"15min":5.4050000000000002},"last":4.0599999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626737,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295612,"kb_used_data":1990196,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91449540,"statfs":{"total":95995035648,"available":93644328960,"internally_reserved":0,"allocated":2037960704,"data_stored":2037443428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":3.6960000000000002,"15min":1.823},"min":{"1min":0.313,"5min":0.245,"15min":0.245},"max":{"1min":2.6880000000000002,"5min":147.25800000000001,"15min":147.25800000000001},"last":2.992},{"interface":"front","average":{"1min":0.94499999999999995,"5min":3.706,"15min":1.8200000000000001},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.254},"max":{"1min":2.8180000000000001,"5min":147.37,"15min":147.37},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:33:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.3440000000000001,"15min":1.042},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.7309999999999999,"5min":8.2690000000000001,"15min":8.2690000000000001},"last":2.9550000000000001},{"interface":"front","average":{"1min":1.298,"5min":1.381,"15min":1.0629999999999999},"min":{"1min":0.33200000000000002,"5min":0.315,"15min":0.28999999999999998},"max":{"1min":6.9770000000000003,"5min":10.099,"15min":10.099},"last":0.44400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:32:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":4.1669999999999998,"15min":2.0230000000000001},"min":{"1min":0.627,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.6910000000000001,"5min":141.93899999999999,"15min":141.93899999999999},"last":2.8559999999999999},{"interface":"front","average":{"1min":0.88,"5min":4.0199999999999996,"15min":1.954},"min":{"1min":0.53600000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.7,"5min":141.898,"15min":141.898},"last":0.57899999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":1.2729999999999999,"15min":1.036},"min":{"1min":0.35899999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.1950000000000001,"5min":10.029,"15min":10.029},"last":3.3069999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.2010000000000001,"15min":1.0940000000000001},"min":{"1min":0.47399999999999998,"5min":0.378,"15min":0.378},"max":{"1min":4.6820000000000004,"5min":6.0519999999999996,"15min":6.0519999999999996},"last":0.63300000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":1.2490000000000001,"15min":1.0329999999999999},"min":{"1min":0.35199999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.2470000000000001,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":2.8929999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.089,"15min":1.0189999999999999},"min":{"1min":0.39000000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.1699999999999999,"5min":11.018000000000001,"15min":11.018000000000001},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.0509999999999999,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.7120000000000002,"5min":3.7120000000000002,"15min":3.9180000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.0670000000000002,"5min":1.242,"15min":1.0780000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:32:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.2390000000000001,"15min":1.123},"min":{"1min":0.499,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.78,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.292,"15min":1.202},"min":{"1min":0.47699999999999998,"5min":0.373,"15min":0.373},"max":{"1min":1.76,"5min":8.9139999999999997,"15min":8.9139999999999997},"last":0.60899999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958659,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1896040,"kb_used_data":1738016,"kb_used_omap":6,"kb_used_meta":158009,"kb_avail":91849112,"statfs":{"total":95995035648,"available":94053490688,"internally_reserved":0,"allocated":1779728384,"data_stored":1779075221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":161801228},"hb_peers":[0,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.1000000000000001,"15min":1.7370000000000001},"min":{"1min":0.23599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.6260000000000003,"5min":9.3130000000000006,"15min":158.435},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.409,"5min":2.3780000000000001,"15min":1.8759999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":4.3209999999999997,"5min":9.9009999999999998,"15min":159.38999999999999},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 04:32:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":3.5950000000000002,"15min":1.6870000000000001},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.056,"5min":157.70400000000001,"15min":157.70400000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.96299999999999997,"5min":3.694,"15min":1.776},"min":{"1min":0.376,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.0519999999999996,"5min":157.64099999999999,"15min":157.64099999999999},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.7010000000000001,"15min":1.123},"min":{"1min":0.41599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1230000000000002,"5min":7.9960000000000004,"15min":7.9960000000000004},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.764,"5min":1.611,"15min":1.0109999999999999},"min":{"1min":0.42999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":5.0270000000000001,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.52300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.627,"15min":1.175},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.9900000000000002,"5min":10.372999999999999,"15min":10.372999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.244,"5min":1.6890000000000001,"15min":1.2629999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.9279999999999999,"5min":9.2850000000000001,"15min":9.2850000000000001},"last":0.85699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.9449999999999998,"5min":3.9449999999999998,"15min":3.9449999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.99099999999999999,"15min":0.98899999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7069999999999999,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.5860000000000001,"15min":1.202},"min":{"1min":0.442,"5min":0.374,"15min":0.374},"max":{"1min":4.2800000000000002,"5min":9.6890000000000001,"15min":9.6890000000000001},"last":2.6110000000000002},{"interface":"front","average":{"1min":1.746,"5min":1.4530000000000001,"15min":1.169},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.9349999999999996,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.32600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.518,"15min":1.2},"min":{"1min":0.41499999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.0270000000000001,"5min":6.3399999999999999,"15min":6.3399999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.6060000000000001,"15min":1.286},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.0250000000000004,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.91200000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122184,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1607988,"kb_used_data":1414188,"kb_used_omap":9,"kb_used_meta":193782,"kb_avail":92137164,"statfs":{"total":95995035648,"available":94348455936,"internally_reserved":0,"allocated":1448128512,"data_stored":1447466373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":198433265},"hb_peers":[1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:33:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6720000000000002,"5min":2.016,"15min":1.089},"min":{"1min":0.33300000000000002,"5min":0.20799999999999999,"15min":0.18099999999999999},"max":{"1min":25.809000000000001,"5min":43.216000000000001,"15min":43.216000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.391,"5min":2.137,"15min":1.1659999999999999},"min":{"1min":0.21199999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":25.931000000000001,"5min":43.091999999999999,"15min":43.091999999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.105,"15min":1.151},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":4.2539999999999996,"5min":37.453000000000003,"15min":37.453000000000003},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.6359999999999999,"15min":1.056},"min":{"1min":0.32900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.2199999999999998,"5min":25.178999999999998,"15min":25.178999999999998},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0700000000000001,"15min":0.89900000000000002},"min":{"1min":0.36599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.0149999999999997,"5min":7.5419999999999998,"15min":7.5419999999999998},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.238,"5min":1.3380000000000001,"15min":0.94699999999999995},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":4.6459999999999999,"5min":7.2270000000000003,"15min":7.5250000000000004},"last":0.72699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.671,"15min":1.256},"min":{"1min":0.315,"5min":0.308,"15min":0.308},"max":{"1min":1.2290000000000001,"5min":9.6590000000000007,"15min":9.6590000000000007},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.1160000000000001,"15min":1.0740000000000001},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.129,"5min":7.5880000000000001,"15min":7.5880000000000001},"last":0.77300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.034,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":3.6339999999999999,"5min":3.6339999999999999,"15min":3.6339999999999999},"last":0.441},{"interface":"front","average":{"1min":1.238,"5min":1.0369999999999999,"15min":1.038},"min":{"1min":0.65500000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":3.343,"5min":3.343,"15min":3.343},"last":2.0150000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.4259999999999999,"15min":1.157},"min":{"1min":0.40799999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.754,"5min":7.508,"15min":7.508},"last":2.1760000000000002},{"interface":"front","average":{"1min":1.002,"5min":1.345,"15min":1.143},"min":{"1min":0.44900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.222,"5min":23.994,"15min":23.994},"last":0.90600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.853,"15min":1.385},"min":{"1min":0.374,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.774,"5min":9.5850000000000009,"15min":10.528},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.929,"15min":1.395},"min":{"1min":0.373,"5min":0.371,"15min":0.371},"max":{"1min":1.1970000000000001,"5min":9.7029999999999994,"15min":9.7029999999999994},"last":1.0029999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795133,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3040088,"kb_used_data":2806608,"kb_used_omap":11,"kb_used_meta":233460,"kb_avail":90705064,"statfs":{"total":95995035648,"available":92881985536,"internally_reserved":0,"allocated":2873966592,"data_stored":2873327590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":239063082},"hb_peers":[0,1,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.375,"15min":0.91300000000000003},"min":{"1min":0.217,"5min":0.191,"15min":0.191},"max":{"1min":2.3340000000000001,"5min":7.577,"15min":7.577},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.4750000000000001,"15min":0.93899999999999995},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.9630000000000001,"5min":8.3529999999999998,"15min":8.3529999999999998},"last":0.48599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:32:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.415,"15min":0.95299999999999996},"min":{"1min":0.317,"5min":0.25,"15min":0.25},"max":{"1min":3.258,"5min":8.1669999999999998,"15min":8.1669999999999998},"last":0.375},{"interface":"front","average":{"1min":0.86399999999999999,"5min":1.663,"15min":1.0389999999999999},"min":{"1min":0.28299999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.4370000000000003,"5min":11.550000000000001,"15min":11.550000000000001},"last":0.52500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":1.296,"15min":1.0009999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0359999999999996,"5min":10.044,"15min":10.044},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.5229999999999999,"15min":1.101},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":6.1130000000000004,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.6040000000000001,"15min":1.198},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.6589999999999998,"5min":7.9210000000000003,"15min":7.9210000000000003},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.984,"15min":1.3069999999999999},"min":{"1min":0.621,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.4889999999999999,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.78100000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0409999999999999,"15min":1.0309999999999999},"min":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.113,"5min":0.98499999999999999,"15min":0.96799999999999997},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":4.649}]},{"osd":6,"last update":"Mon Oct 9 04:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.9239999999999999,"15min":1.3360000000000001},"min":{"1min":0.68600000000000005,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.3949999999999996,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":4.734},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.2450000000000001,"15min":1.1100000000000001},"min":{"1min":0.64300000000000002,"5min":0.377,"15min":0.377},"max":{"1min":2.4140000000000001,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.33400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.5529999999999999,"15min":1.2569999999999999},"min":{"1min":0.67300000000000004,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.669,"5min":8.2430000000000003,"15min":8.2430000000000003},"last":0.751},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.599,"15min":1.2649999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0699999999999998,"5min":8.952,"15min":8.952},"last":0.82099999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631605,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3657560,"kb_used_data":3495504,"kb_used_omap":5,"kb_used_meta":162042,"kb_avail":90087592,"statfs":{"total":95995035648,"available":92249694208,"internally_reserved":0,"allocated":3579396096,"data_stored":3578756750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":165931182},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":2.2090000000000001,"15min":1.2889999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":5.0309999999999997,"5min":61.966999999999999,"15min":61.966999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.369,"5min":2.2879999999999998,"15min":1.28},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.1890000000000001,"5min":62.052999999999997,"15min":62.052999999999997},"last":0.37}]},{"osd":1,"last update":"Mon Oct 9 04:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.899,"15min":1.121},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.2439999999999998,"5min":61.395000000000003,"15min":61.395000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.5309999999999999,"5min":1.821,"15min":1.1579999999999999},"min":{"1min":0.28399999999999997,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.5750000000000002,"5min":61.179000000000002,"15min":61.179000000000002},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":3.827,"15min":1.851},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":6.2699999999999996,"5min":181.959,"15min":181.959},"last":0.501},{"interface":"front","average":{"1min":2.1560000000000001,"5min":3.7570000000000001,"15min":1.905},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.6600000000000001,"5min":181.91200000000001,"15min":181.91200000000001},"last":0.48599999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.038,"15min":1.054},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.609,"5min":6.0190000000000001,"15min":6.0190000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2370000000000001,"15min":1.081},"min":{"1min":0.44900000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.3810000000000002,"5min":7.0739999999999998,"15min":7.0739999999999998},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.96499999999999997,"15min":1.0229999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":1.917},{"interface":"front","average":{"1min":0.872,"5min":0.997,"15min":1.056},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.996,"5min":1.996,"15min":1.996},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.01,"15min":1.0249999999999999},"min":{"1min":0.42199999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.3019999999999996,"5min":7.1120000000000001,"15min":7.1120000000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.0149999999999999,"15min":1.0609999999999999},"min":{"1min":0.48899999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.6259999999999999,"5min":6.0039999999999996,"15min":6.0039999999999996},"last":0.746}]},{"osd":7,"last update":"Mon Oct 9 04:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.214,"15min":1.147},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.1749999999999998,"5min":6.1749999999999998,"15min":6.1749999999999998},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.167,"15min":1.2010000000000001},"min":{"1min":0.499,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6349999999999998,"5min":6.8209999999999997,"15min":6.8209999999999997},"last":0.65700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019206,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1853780,"kb_used_data":1544268,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91891372,"statfs":{"total":95995035648,"available":94096764928,"internally_reserved":0,"allocated":1581330432,"data_stored":1580811538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.5820000000000001,"15min":1.2769999999999999},"min":{"1min":0.26400000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.4119999999999999,"5min":20.721,"15min":48.450000000000003},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.7889999999999999,"15min":1.383},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.1320000000000001,"5min":21.061,"15min":49.283000000000001},"last":0.72099999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:33:06 2023","interfaces":[{"interface":"back","average":{"1min":5.2439999999999998,"5min":2.3399999999999999,"15min":1.4199999999999999},"min":{"1min":0.31900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":79.361999999999995,"5min":79.361999999999995,"15min":79.361999999999995},"last":0.79700000000000004},{"interface":"front","average":{"1min":5.3869999999999996,"5min":2.6070000000000002,"15min":1.5189999999999999},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":79.292000000000002,"5min":79.292000000000002,"15min":79.292000000000002},"last":0.68799999999999994}]},{"osd":2,"last update":"Mon Oct 9 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.0619999999999998,"15min":1.3129999999999999},"min":{"1min":0.317,"5min":0.31,"15min":0.31},"max":{"1min":3.5019999999999998,"5min":48.323,"15min":48.323},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.86899999999999999,"5min":2.5369999999999999,"15min":1.48},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":2.831,"5min":48.289000000000001,"15min":48.289000000000001},"last":0.76600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.2370000000000001,"15min":1.105},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":8.516,"5min":8.516,"15min":8.516},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.728,"5min":1.5269999999999999,"15min":1.2110000000000001},"min":{"1min":0.41699999999999998,"5min":0.34799999999999998,"15min":0.32000000000000001},"max":{"1min":8.4920000000000009,"5min":11.112,"15min":11.112},"last":0.84899999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98999999999999999,"15min":0.95399999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":3.5409999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.087,"15min":0.95399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":1.4770000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.95599999999999996,"15min":0.97599999999999998},"min":{"1min":0.26700000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.7639999999999998,"5min":5.3550000000000004,"15min":5.3550000000000004},"last":1.4490000000000001},{"interface":"front","average":{"1min":0.875,"5min":1.4219999999999999,"15min":1.175},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.589,"5min":15.977,"15min":15.977},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.347,"15min":1.24},"min":{"1min":0.41899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0870000000000002,"5min":11.202999999999999,"15min":11.725},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.177,"15min":1.151},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.1230000000000002,"5min":6.5730000000000004,"15min":6.5730000000000004},"last":1.0169999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822912,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3745056,"kb_used_data":3439640,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90000096,"statfs":{"total":95995035648,"available":92160098304,"internally_reserved":0,"allocated":3522191360,"data_stored":3521698716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8919999999999999,"5min":1.474,"15min":1.073},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":53.731000000000002,"5min":53.731000000000002,"15min":53.731000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":1.5429999999999999,"15min":1.0620000000000001},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":53.222999999999999,"5min":53.222999999999999,"15min":53.222999999999999},"last":2.359}]},{"osd":1,"last update":"Mon Oct 9 04:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.98099999999999998,"15min":0.997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.252},{"interface":"front","average":{"1min":1.038,"5min":0.98599999999999999,"15min":0.96199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.4649999999999999,"5min":2.4649999999999999,"15min":2.4649999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:26 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.94699999999999995,"15min":0.94099999999999995},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":2.4430000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.0169999999999999,"15min":1.0209999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":2.5390000000000001,"5min":2.5390000000000001,"15min":2.5390000000000001},"last":2.4649999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.008,"15min":0.99399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":2.2789999999999999,"5min":2.2789999999999999,"15min":2.2789999999999999},"last":0.378},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.1240000000000001,"15min":1.0740000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.5140000000000002,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":2.4180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:43 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96299999999999997,"15min":0.94899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9100000000000001,"5min":3.9100000000000001,"15min":3.9100000000000001},"last":5.6479999999999997},{"interface":"front","average":{"1min":1.304,"5min":1.0900000000000001,"15min":1.012},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":3.9990000000000001},"last":2.395}]},{"osd":6,"last update":"Mon Oct 9 04:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.998,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.9849999999999999,"5min":3.9849999999999999,"15min":3.9849999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0740000000000001,"15min":1.0660000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.847,"5min":1.847,"15min":2.5840000000000001},"last":0.32800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.1379999999999999,"15min":1.113},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.431,"5min":3.992,"15min":3.992},"last":0.312},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.1120000000000001,"15min":1.095},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":1.4610000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1445986304,"data_stored":1445986304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1778188288,"data_stored":1778188288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2868248576,"data_stored":2868248576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3577872384,"data_stored":3577872384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1426128896,"data_stored":1426128896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2039349248,"data_stored":2039349248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2128674816,"data_stored":2128674816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:33:38.161 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T04:33:38.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:38.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:38.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:38.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:38.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:38.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:38.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:38.499 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:33:38.522 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:33:38.543 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:33:38.564 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:33:38.588 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:33:38.597 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:38.611 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-09T04:33:38.612 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:33:38.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:38.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:38.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:38.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:38.859 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:38 smithi082 bash[77063]: audit 2023-10-09T04:33:38.473350+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.82:0/3638172108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:38.859 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:38 smithi082 bash[79733]: audit 2023-10-09T04:33:38.473350+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.82:0/3638172108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:38.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 16 87335 87319 61.3047 0 - 0.0110854 2023-10-09T04:33:38.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:38.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:38.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:38 smithi153 bash[59565]: audit 2023-10-09T04:33:38.473350+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.82:0/3638172108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:38.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:39.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:39.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:39.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:39.381 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:39.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.552 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:39.657 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:39.664 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:39 smithi082 bash[79733]: cluster 2023-10-09T04:33:38.794325+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v545: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:39.665 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:39 smithi082 bash[77063]: cluster 2023-10-09T04:33:38.794325+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v545: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:39.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:39 smithi153 bash[59565]: cluster 2023-10-09T04:33:38.794325+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v545: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:39.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:39.785 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:39.860 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:39] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.33.4" 2023-10-09T04:33:39.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 90 16 87335 87319 60.6236 0 - 0.0110854 2023-10-09T04:33:39.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:39.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:40.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:40.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:40.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:40.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:40.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:40.336 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:40.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:40.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:40.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:33:40.475 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:33:40.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:40.600 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:40.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:40.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:40.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:40.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:40.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 91 16 87335 87319 59.9575 0 - 0.0110854 2023-10-09T04:33:40.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:40.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:40.949 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:40.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:41.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:41.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:41.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:41.386 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:41.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:41.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:41.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:41.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:41.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:41.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:41.693 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:41.797 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:41.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:41.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:41.868 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:33:41.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 92 16 87335 87319 59.3059 0 - 0.0110854 2023-10-09T04:33:41.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:41.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:41.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:41.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:42.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[77063]: cluster 2023-10-09T04:33:40.795791+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v546: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[77063]: audit 2023-10-09T04:33:40.910649+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.82:0/4155415164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[77063]: audit 2023-10-09T04:33:40.911185+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[79733]: cluster 2023-10-09T04:33:40.795791+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v546: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[79733]: audit 2023-10-09T04:33:40.910649+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.82:0/4155415164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.066 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:41 smithi082 bash[79733]: audit 2023-10-09T04:33:40.911185+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:41 smithi153 bash[59565]: cluster 2023-10-09T04:33:40.795791+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v546: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:41 smithi153 bash[59565]: audit 2023-10-09T04:33:40.910649+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.82:0/4155415164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:41 smithi153 bash[59565]: audit 2023-10-09T04:33:40.911185+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:42.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:42.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:42.456 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:33:42.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:42.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:42.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:42.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:42.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:42.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 93 16 87335 87319 58.6682 0 - 0.0110854 2023-10-09T04:33:42.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:42.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:42.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[77063]: cluster 2023-10-09T04:33:41.850049+0000 mon.a (mon.0) 848 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:42.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[77063]: audit 2023-10-09T04:33:41.862546+0000 mon.a (mon.0) 849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:42.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[77063]: cluster 2023-10-09T04:33:41.862692+0000 mon.a (mon.0) 850 : cluster 0 osdmap e142: 8 total, 6 up, 8 in 2023-10-09T04:33:42.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[77063]: cluster 2023-10-09T04:33:41.882211+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:42.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[77063]: cluster 2023-10-09T04:33:41.882308+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[79733]: cluster 2023-10-09T04:33:41.850049+0000 mon.a (mon.0) 848 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[79733]: audit 2023-10-09T04:33:41.862546+0000 mon.a (mon.0) 849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[79733]: cluster 2023-10-09T04:33:41.862692+0000 mon.a (mon.0) 850 : cluster 0 osdmap e142: 8 total, 6 up, 8 in 2023-10-09T04:33:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[79733]: cluster 2023-10-09T04:33:41.882211+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:42.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:42 smithi082 bash[79733]: cluster 2023-10-09T04:33:41.882308+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:43.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:43.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:43.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:42 smithi153 bash[59565]: cluster 2023-10-09T04:33:41.850049+0000 mon.a (mon.0) 848 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:43.123 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:42 smithi153 bash[59565]: audit 2023-10-09T04:33:41.862546+0000 mon.a (mon.0) 849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:43.123 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:42 smithi153 bash[59565]: cluster 2023-10-09T04:33:41.862692+0000 mon.a (mon.0) 850 : cluster 0 osdmap e142: 8 total, 6 up, 8 in 2023-10-09T04:33:43.123 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:42 smithi153 bash[59565]: cluster 2023-10-09T04:33:41.882211+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:43.123 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:42 smithi153 bash[59565]: cluster 2023-10-09T04:33:41.882308+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T04:33:43.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:43.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:43.382 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:43.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:43.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:43.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:43.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:43.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:43.679 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:43.756 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:43.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:43.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 94 16 87335 87319 58.0442 0 - 0.0110854 2023-10-09T04:33:43.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:43.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:44.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.066 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:43 smithi082 bash[77063]: cluster 2023-10-09T04:33:42.796734+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v548: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:44.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:43 smithi082 bash[79733]: cluster 2023-10-09T04:33:42.796734+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v548: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:44.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:43 smithi153 bash[59565]: cluster 2023-10-09T04:33:42.796734+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v548: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:44.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:44.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:44.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:44.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:44.446 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.469 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:44.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:44.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:44.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:44.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:44.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:44.873 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 95 16 87335 87319 57.4333 0 - 0.0110854 2023-10-09T04:33:44.957 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:44.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:44.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:44.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:45.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:45.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:45.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:45.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:45.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:45.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:33:45.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:45.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[77063]: audit 2023-10-09T04:33:44.269730+0000 mon.a (mon.0) 851 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[77063]: audit 2023-10-09T04:33:44.270810+0000 mon.a (mon.0) 852 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[77063]: cluster 2023-10-09T04:33:44.797791+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v549: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[79733]: audit 2023-10-09T04:33:44.269730+0000 mon.a (mon.0) 851 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[79733]: audit 2023-10-09T04:33:44.270810+0000 mon.a (mon.0) 852 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:45 smithi082 bash[79733]: cluster 2023-10-09T04:33:44.797791+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v549: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:45.484 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:33:45.484 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:33:45.484 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 0 2023-10-09T04:33:45.484 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:33:45.494 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:45.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:45.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:45.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:45 smithi153 bash[59565]: audit 2023-10-09T04:33:44.269730+0000 mon.a (mon.0) 851 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:33:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:45 smithi153 bash[59565]: audit 2023-10-09T04:33:44.270810+0000 mon.a (mon.0) 852 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:45 smithi153 bash[59565]: cluster 2023-10-09T04:33:44.797791+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v549: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:45.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:45.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:45.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 96 16 87335 87319 56.8351 0 - 0.0110854 2023-10-09T04:33:45.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:45.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:45.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:46.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:46.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:46.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:46.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:46.284 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:46.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:46.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:46.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:46.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:46.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:46.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:46.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:46.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:46.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:33:46.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:46.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 97 16 87335 87319 56.2492 0 - 0.0110854 2023-10-09T04:33:46.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:46.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:47.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:47.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:47.224 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:47.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:47.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:47.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:47.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:47.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:47.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:47.801 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:47.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:47.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 98 16 87335 87319 55.6753 0 - 0.0110854 2023-10-09T04:33:47.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:47.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:47.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:48.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:47 smithi153 bash[59565]: cluster 2023-10-09T04:33:46.798838+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v550: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:48.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:48.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:47 smithi082 bash[77063]: cluster 2023-10-09T04:33:46.798838+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v550: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:48.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:47 smithi082 bash[79733]: cluster 2023-10-09T04:33:46.798838+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v550: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:48.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:48.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:48.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:48.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:48.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:48.679 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:48.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:48.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:48.809 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:48.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:48.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 99 16 87335 87319 55.113 0 - 0.0110854 2023-10-09T04:33:48.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:48.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:49.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:49.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:49.173 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:49.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:49.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.339 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:49.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:49.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:49.601 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:49.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:49.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:49.872 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 6 pgp_num to 16 2023-10-09T04:33:49.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:49.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:33:49.873718+0000 min lat: 0.00116931 max lat: 1.03787 avg lat: 0.0110854 2023-10-09T04:33:49.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:33:49.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 100 16 87335 87319 54.5619 0 - 0.0110854 2023-10-09T04:33:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:49.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:49.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.996 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:50.005 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:49] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T04:33:50.100 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:50.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:49 smithi153 bash[59565]: cluster 2023-10-09T04:33:48.799771+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v551: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:50.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:49 smithi153 bash[59565]: audit 2023-10-09T04:33:49.017470+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:33:50.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:50.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:50.270 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:50.271 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[77063]: cluster 2023-10-09T04:33:48.799771+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v551: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:50.272 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[77063]: audit 2023-10-09T04:33:49.017470+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:33:50.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[79733]: cluster 2023-10-09T04:33:48.799771+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v551: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:50.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[79733]: audit 2023-10-09T04:33:49.017470+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:33:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:50.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:50.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:50.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:50.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:50.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:50.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:50.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:50.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:50.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 101 16 87335 87319 54.0218 0 - 0.0110854 2023-10-09T04:33:50.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:50.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:50 smithi153 bash[59565]: cluster 2023-10-09T04:33:49.865027+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T04:33:50.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:50 smithi153 bash[59565]: audit 2023-10-09T04:33:49.866957+0000 mon.a (mon.0) 854 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:33:50.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:50 smithi153 bash[59565]: cluster 2023-10-09T04:33:49.867037+0000 mon.a (mon.0) 855 : cluster 0 osdmap e143: 8 total, 6 up, 8 in 2023-10-09T04:33:50.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:50.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:51.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:51.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:51.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:51.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:51.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[79733]: cluster 2023-10-09T04:33:49.865027+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T04:33:51.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[79733]: audit 2023-10-09T04:33:49.866957+0000 mon.a (mon.0) 854 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:33:51.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[79733]: cluster 2023-10-09T04:33:49.867037+0000 mon.a (mon.0) 855 : cluster 0 osdmap e143: 8 total, 6 up, 8 in 2023-10-09T04:33:51.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[77063]: cluster 2023-10-09T04:33:49.865027+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T04:33:51.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[77063]: audit 2023-10-09T04:33:49.866957+0000 mon.a (mon.0) 854 : audit 1 from='client.? 172.21.15.82:0/504441969' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:33:51.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:50 smithi082 bash[77063]: cluster 2023-10-09T04:33:49.867037+0000 mon.a (mon.0) 855 : cluster 0 osdmap e143: 8 total, 6 up, 8 in 2023-10-09T04:33:51.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:51.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:51.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:51.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:51.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:51.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:51.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:51.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:51.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:51.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:51.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 102 16 87335 87319 53.4922 0 - 0.0110854 2023-10-09T04:33:51.955 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:51.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:51.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:51.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:52.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:52.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:51 smithi153 bash[59565]: cluster 2023-10-09T04:33:50.801024+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v553: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:52.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:51 smithi082 bash[77063]: cluster 2023-10-09T04:33:50.801024+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v553: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:51 smithi082 bash[79733]: cluster 2023-10-09T04:33:50.801024+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v553: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:52.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:52.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:52.310 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:52.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:52.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:52.490 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:52.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:52.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:52.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:52.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:52.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:52.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 103 16 87335 87319 52.9729 0 - 0.0110854 2023-10-09T04:33:52.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:52.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:52.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:53.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:53.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:53.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:53.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:52 smithi082 bash[79733]: cluster 2023-10-09T04:33:52.802326+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v554: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:53.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:52 smithi082 bash[77063]: cluster 2023-10-09T04:33:52.802326+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v554: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:52 smithi153 bash[59565]: cluster 2023-10-09T04:33:52.802326+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v554: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:53.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:53.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:53.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:53.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:53.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:53.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:53.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:53.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:53.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:53.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:53.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:53.874 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 104 16 87335 87319 52.4636 0 - 0.0110854 2023-10-09T04:33:53.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:53.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:53.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:54.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:54.080 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:54.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:54.244 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:33:54.266 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:33:54.288 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:33:54.308 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:33:54.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:54.337 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:33:54.363 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-09T04:33:54.364 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:33:54.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:54 smithi153 bash[59565]: audit 2023-10-09T04:33:54.214323+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.82:0/780127942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:54.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:54.449 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:54 smithi082 bash[77063]: audit 2023-10-09T04:33:54.214323+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.82:0/780127942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:54.449 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:54 smithi082 bash[79733]: audit 2023-10-09T04:33:54.214323+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.82:0/780127942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:54.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:54.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:54.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:54.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:54.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:54.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:54.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 105 16 87335 87319 51.964 0 - 0.0110854 2023-10-09T04:33:54.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:54.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:54.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:55.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:55.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:55.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:55.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:55.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:55.494 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.524 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:55.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:55.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:55.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:55.762 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:55 smithi082 bash[77063]: cluster 2023-10-09T04:33:54.803201+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v555: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:55.762 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:55 smithi082 bash[79733]: cluster 2023-10-09T04:33:54.803201+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v555: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:55.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:55.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 106 16 87335 87319 51.4738 0 - 0.0110854 2023-10-09T04:33:55.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:55.898 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:55 smithi153 bash[59565]: cluster 2023-10-09T04:33:54.803201+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v555: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:55.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:55.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:33:56.011 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:33:56.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:56.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:56.157 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:56.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:56.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:56.312 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:56.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:56.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:56.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:56.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:56.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:56.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:56.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:56.825 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5, 6] live_osds: [0, 1, 2, 3, 4, 7] 2023-10-09T04:33:56.825 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-10-09T04:33:56.825 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-10-09T04:33:56.826 DEBUG:teuthology.orchestra.run.smithi153:osd.6> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T04:33:56.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl start ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T04:33:56.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:56.844 INFO:journalctl@ceph.osd.6.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:33:56.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:33:56.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 107 16 87335 87319 50.9928 0 - 0.0110854 2023-10-09T04:33:56.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:56.938 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:56 smithi153 systemd[1]: Started Ceph osd.6 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:33:56.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:56.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:56.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:57.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:57.083 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:57.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:57.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:57.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:57.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:57.410 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:57.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:57.544 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:57.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:57.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:57.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:57.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:57.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:57.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 108 16 87335 87319 50.5207 0 - 0.0110854 2023-10-09T04:33:57.906 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:57.935 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:57.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:58.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:33:58.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:58.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:58.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:57 smithi153 bash[59565]: cluster 2023-10-09T04:33:56.804162+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v556: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:58.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.198 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:57 smithi082 bash[77063]: cluster 2023-10-09T04:33:56.804162+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v556: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:58.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:57 smithi082 bash[79733]: cluster 2023-10-09T04:33:56.804162+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v556: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:33:58.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:58.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:58.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:58.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:58.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:58.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:58.727 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:58.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-09T04:33:58.839 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:33:58 smithi153 bash[81091]: --> ceph-volume raw activate successful for osd ID: 6 2023-10-09T04:33:58.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:58.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 109 16 87335 87319 50.0573 0 - 0.0110854 2023-10-09T04:33:58.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:58.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:58.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:58.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:59.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:59.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:59.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:59.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:59.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:59.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:59.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:59.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:59.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:59.741 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:59.741 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:59.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.766 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:33:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:33:59] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-09T04:33:59.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:59.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 110 16 87335 87319 49.6023 0 - 0.0110854 2023-10-09T04:33:59.885 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:33:59.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:00.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:00.031 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:59 smithi082 bash[77063]: cluster 2023-10-09T04:33:58.805403+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v557: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:00.031 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:33:59 smithi082 bash[77063]: audit 2023-10-09T04:33:59.266416+0000 mon.a (mon.0) 856 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:00.031 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:59 smithi082 bash[79733]: cluster 2023-10-09T04:33:58.805403+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v557: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:00.031 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:33:59 smithi082 bash[79733]: audit 2023-10-09T04:33:59.266416+0000 mon.a (mon.0) 856 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:00.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:00.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:59 smithi153 bash[59565]: cluster 2023-10-09T04:33:58.805403+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v557: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:33:59 smithi153 bash[59565]: audit 2023-10-09T04:33:59.266416+0000 mon.a (mon.0) 856 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:00.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:00.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:00.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:00.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:00.447 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:00.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:34:00.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:00.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:00.676 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:34:00 smithi153 bash[81466]: debug 2023-10-09T04:34:00.487+0000 7f436a91f700 1 -- 172.21.15.153:0/3877402201 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5587ba3764e0 con 0x5587ba3e9400 2023-10-09T04:34:00.711 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:00.742 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:00.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:00.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 111 16 87335 87319 49.1554 0 - 0.0110854 2023-10-09T04:34:00.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:00.889 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:34:00.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:01.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:00 smithi082 bash[77063]: audit 2023-10-09T04:33:59.961417+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:01.115 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:00 smithi082 bash[79733]: audit 2023-10-09T04:33:59.961417+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:01.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:00 smithi153 bash[59565]: audit 2023-10-09T04:33:59.961417+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:01.134 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[81466]: debug 2023-10-09T04:34:01.051+0000 7f437239a200 -1 Falling back to public interface 2023-10-09T04:34:01.153 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:01.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:01.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:01.391 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:01.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:01.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:01.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:01.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:01.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 112 16 87335 87319 48.7166 0 - 0.0110854 2023-10-09T04:34:01.930 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:01.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:01.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[77063]: cluster 2023-10-09T04:34:00.806808+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v558: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[77063]: cluster 2023-10-09T04:34:00.871189+0000 mon.a (mon.0) 858 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[77063]: audit 2023-10-09T04:34:00.883520+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[77063]: cluster 2023-10-09T04:34:00.883603+0000 mon.a (mon.0) 860 : cluster 0 osdmap e144: 8 total, 6 up, 8 in 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[77063]: cluster 2023-10-09T04:34:01.076557+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[79733]: cluster 2023-10-09T04:34:00.806808+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v558: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[79733]: cluster 2023-10-09T04:34:00.871189+0000 mon.a (mon.0) 858 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[79733]: audit 2023-10-09T04:34:00.883520+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[79733]: cluster 2023-10-09T04:34:00.883603+0000 mon.a (mon.0) 860 : cluster 0 osdmap e144: 8 total, 6 up, 8 in 2023-10-09T04:34:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:01 smithi082 bash[79733]: cluster 2023-10-09T04:34:01.076557+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:02.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:02.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[59565]: cluster 2023-10-09T04:34:00.806808+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v558: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[59565]: cluster 2023-10-09T04:34:00.871189+0000 mon.a (mon.0) 858 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[59565]: audit 2023-10-09T04:34:00.883520+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.82:0/2675521960' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[59565]: cluster 2023-10-09T04:34:00.883603+0000 mon.a (mon.0) 860 : cluster 0 osdmap e144: 8 total, 6 up, 8 in 2023-10-09T04:34:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:01 smithi153 bash[59565]: cluster 2023-10-09T04:34:01.076557+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:02.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:02.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:02.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:02.481 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:02.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:02.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:02.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:02.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:02.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:02.843 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:34:02.875 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 113 16 87335 87319 48.2855 0 - 0.0110854 2023-10-09T04:34:02.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:02.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:02.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:03.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.035 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:02 smithi153 bash[59565]: cluster 2023-10-09T04:34:01.541462+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:03.035 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:02 smithi153 bash[59565]: cluster 2023-10-09T04:34:02.808132+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v560: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:03.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:03.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.183 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:03.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:02 smithi082 bash[77063]: cluster 2023-10-09T04:34:01.541462+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:03.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:02 smithi082 bash[77063]: cluster 2023-10-09T04:34:02.808132+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v560: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:03.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:02 smithi082 bash[79733]: cluster 2023-10-09T04:34:01.541462+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T04:34:03.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:02 smithi082 bash[79733]: cluster 2023-10-09T04:34:02.808132+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v560: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:03.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:03.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:03.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:03.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:03.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:03.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:03.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:34:03.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:03.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:03.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 114 16 87335 87319 47.862 0 - 0.0110854 2023-10-09T04:34:03.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:03.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:03.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:03.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:04.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:04.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:04.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:04.241 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:04.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:04.456 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:04.487 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:04.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:04.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:04.723 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:04.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:04.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:04.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 115 16 87335 87319 47.4458 0 - 0.0110854 2023-10-09T04:34:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:04.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:04.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:05.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:05.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:05.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:05.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:05.255 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:05.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:05.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:05.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:05.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:05.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:05.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:05.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:05.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:05.761 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:05.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:05.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:05 smithi153 bash[59565]: cluster 2023-10-09T04:34:04.809518+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v561: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:05.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 116 16 87335 87319 47.0369 0 - 0.0110854 2023-10-09T04:34:05.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:05.912 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:05.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:05.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:05.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:05 smithi082 bash[77063]: cluster 2023-10-09T04:34:04.809518+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v561: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:05.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:05 smithi082 bash[79733]: cluster 2023-10-09T04:34:04.809518+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v561: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:05.991 INFO:teuthology.orchestra.run.smithi153.stderr:no valid command found; 10 closest matches: 2023-10-09T04:34:05.991 INFO:teuthology.orchestra.run.smithi153.stderr:0 2023-10-09T04:34:05.991 INFO:teuthology.orchestra.run.smithi153.stderr:1 2023-10-09T04:34:05.991 INFO:teuthology.orchestra.run.smithi153.stderr:2 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:abort 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:assert 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs debug_inject_read_zeros 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs files list 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs stats 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:bluestore allocator dump block 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:bluestore allocator fragmentation block 2023-10-09T04:34:05.992 INFO:teuthology.orchestra.run.smithi153.stderr:admin_socket: invalid command 2023-10-09T04:34:06.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:06.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:06.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:06.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:06.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:06.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:06.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:06.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:06.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:06.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:34:06.726 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-10-09T04:34:06.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:06.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:06.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 117 16 87335 87319 46.6349 0 - 0.0110854 2023-10-09T04:34:06.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:06.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:06.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:07.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:07.084 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:07.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:07.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:07.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:07.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:07.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:07.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:07.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:07.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:07.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:07.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 118 16 87335 87319 46.2397 0 - 0.0110854 2023-10-09T04:34:07.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:07.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:07.901 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:34:07 smithi153 bash[81466]: debug 2023-10-09T04:34:07.643+0000 7f437239a200 -1 osd.6 138 log_to_monitors {default=true} 2023-10-09T04:34:07.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:07 smithi153 bash[59565]: cluster 2023-10-09T04:34:06.810845+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v562: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:07.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:07 smithi153 bash[59565]: audit 2023-10-09T04:34:07.650227+0000 mon.b (mon.1) 7 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:07.901 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:07 smithi153 bash[59565]: audit 2023-10-09T04:34:07.651351+0000 mon.a (mon.0) 861 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:07.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:08.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:08.115 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:08.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:08.176 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 04:34:07 smithi153 bash[81466]: debug 2023-10-09T04:34:07.895+0000 7f4364896700 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:34:08.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:08.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[77063]: cluster 2023-10-09T04:34:06.810845+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v562: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:08.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[77063]: audit 2023-10-09T04:34:07.650227+0000 mon.b (mon.1) 7 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:08.247 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[77063]: audit 2023-10-09T04:34:07.651351+0000 mon.a (mon.0) 861 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:08.248 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[79733]: cluster 2023-10-09T04:34:06.810845+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v562: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:08.248 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[79733]: audit 2023-10-09T04:34:07.650227+0000 mon.b (mon.1) 7 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:08.248 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:07 smithi082 bash[79733]: audit 2023-10-09T04:34:07.651351+0000 mon.a (mon.0) 861 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:34:08.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:08.301 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:08.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:08.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:08.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:08.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:08.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:08.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:08.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 119 16 87335 87319 45.8512 0 - 0.0110854 2023-10-09T04:34:08.885 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.895 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[79733]: audit 2023-10-09T04:34:07.870600+0000 mon.a (mon.0) 862 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:34:08.895 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[79733]: cluster 2023-10-09T04:34:07.870691+0000 mon.a (mon.0) 863 : cluster 0 osdmap e145: 8 total, 6 up, 8 in 2023-10-09T04:34:08.895 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[79733]: audit 2023-10-09T04:34:07.874074+0000 mon.b (mon.1) 8 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:08.895 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[79733]: audit 2023-10-09T04:34:07.879361+0000 mon.a (mon.0) 864 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.811567+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v564: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[77063]: audit 2023-10-09T04:34:07.870600+0000 mon.a (mon.0) 862 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[77063]: cluster 2023-10-09T04:34:07.870691+0000 mon.a (mon.0) 863 : cluster 0 osdmap e145: 8 total, 6 up, 8 in 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[77063]: audit 2023-10-09T04:34:07.874074+0000 mon.b (mon.1) 8 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[77063]: audit 2023-10-09T04:34:07.879361+0000 mon.a (mon.0) 864 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:08.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:08 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.811567+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v564: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:08.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:08.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:09.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:09.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:09.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:09.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:08 smithi153 bash[59565]: audit 2023-10-09T04:34:07.870600+0000 mon.a (mon.0) 862 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:34:09.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:08 smithi153 bash[59565]: cluster 2023-10-09T04:34:07.870691+0000 mon.a (mon.0) 863 : cluster 0 osdmap e145: 8 total, 6 up, 8 in 2023-10-09T04:34:09.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:08 smithi153 bash[59565]: audit 2023-10-09T04:34:07.874074+0000 mon.b (mon.1) 8 : audit 1 from='osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:09.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:08 smithi153 bash[59565]: audit 2023-10-09T04:34:07.879361+0000 mon.a (mon.0) 864 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:09.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:08 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.811567+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v564: 89 pgs: 1 active+clean+remapped, 38 active+clean, 5 undersized+peered, 27 active+undersized, 6 undersized+degraded+peered, 12 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60186/262044 objects degraded (22.968%) 2023-10-09T04:34:09.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:09.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:09.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:09.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:09.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:09.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:09.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:09.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:09.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:09.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:34:09.876064+0000 min lat: 0.00116931 max lat: 1.03787 avg lat: 0.0110854 2023-10-09T04:34:09.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:34:09.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 120 16 87335 87319 45.4691 0 - 0.0110854 2023-10-09T04:34:09.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.688857+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:09.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.694709+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:09.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.697346+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:09.897 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:09] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-09T04:34:09.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:09.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:09.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:10.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:10.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.875151+0000 mon.a (mon.0) 865 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.878451+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.880713+0000 mon.a (mon.0) 866 : cluster 1 osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791] boot 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.880772+0000 mon.a (mon.0) 867 : cluster 0 osdmap e146: 8 total, 7 up, 8 in 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.881575+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: audit 2023-10-09T04:34:08.887548+0000 mon.a (mon.0) 868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.884395+0000 mon.a (mon.0) 869 : cluster 0 osdmap e147: 8 total, 7 up, 8 in 2023-10-09T04:34:10.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.688857+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.694709+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.697346+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.875151+0000 mon.a (mon.0) 865 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.878451+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.880713+0000 mon.a (mon.0) 866 : cluster 1 osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791] boot 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.880772+0000 mon.a (mon.0) 867 : cluster 0 osdmap e146: 8 total, 7 up, 8 in 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.881575+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: audit 2023-10-09T04:34:08.887548+0000 mon.a (mon.0) 868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:34:10.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:09 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.884395+0000 mon.a (mon.0) 869 : cluster 0 osdmap e147: 8 total, 7 up, 8 in 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.688857+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.694709+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.697346+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.875151+0000 mon.a (mon.0) 865 : cluster 3 Health check update: 1 osds down (OSD_DOWN) 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.878451+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.880713+0000 mon.a (mon.0) 866 : cluster 1 osd.6 [v2:172.21.15.153:6808/2741586791,v1:172.21.15.153:6809/2741586791] boot 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.880772+0000 mon.a (mon.0) 867 : cluster 0 osdmap e146: 8 total, 7 up, 8 in 2023-10-09T04:34:10.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.881575+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T04:34:10.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: audit 2023-10-09T04:34:08.887548+0000 mon.a (mon.0) 868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:34:10.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:09 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.884395+0000 mon.a (mon.0) 869 : cluster 0 osdmap e147: 8 total, 7 up, 8 in 2023-10-09T04:34:10.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:10.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:10.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:10.483 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:10.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:10.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:10.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:10.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:10.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:10.876 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 120.18 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 87335 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 45.4188 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 50.9506 2023-10-09T04:34:10.877 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 188.062 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 726 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 815.21 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 3009 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 0 2023-10-09T04:34:10.878 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.02201 2023-10-09T04:34:10.879 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.807213 2023-10-09T04:34:10.879 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 60.2887 2023-10-09T04:34:10.879 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00116931 2023-10-09T04:34:10.884 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:34:10.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.688840+0000 osd.6 (osd.6) 1 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.880701+0000 osd.6 (osd.6) 2 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:08.886586+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.880464+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.886006+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.886006+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.888941+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.985 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:09.890318+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[77063]: cluster 2023-10-09T04:34:10.813152+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v567: 89 pgs: 1 active+recovering+degraded, 7 peering, 1 active+clean+remapped, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43998/262044 objects degraded (16.790%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.688840+0000 osd.6 (osd.6) 1 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.880701+0000 osd.6 (osd.6) 2 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:08.886586+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.880464+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.886006+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.886006+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.888941+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:09.890318+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:10.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:10 smithi082 bash[79733]: cluster 2023-10-09T04:34:10.813152+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v567: 89 pgs: 1 active+recovering+degraded, 7 peering, 1 active+clean+remapped, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43998/262044 objects degraded (16.790%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:11.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:11.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:11.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.688840+0000 osd.6 (osd.6) 1 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.880701+0000 osd.6 (osd.6) 2 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:08.886586+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.880464+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.886006+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.886006+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.888941+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:09.890318+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T04:34:11.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:10 smithi153 bash[59565]: cluster 2023-10-09T04:34:10.813152+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v567: 89 pgs: 1 active+recovering+degraded, 7 peering, 1 active+clean+remapped, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43998/262044 objects degraded (16.790%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:11.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:11.373 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:34:11.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:11.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:11.425 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:34:11.480 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:34:11.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:11.539 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:34:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:11.577 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:34:11.617 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:34:11.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.647 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:34:11.648 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.648 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:34:11.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:11.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:34:11.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:11.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:11.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:11.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[77063]: cluster 2023-10-09T04:34:10.894123+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:34:11.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[77063]: cluster 2023-10-09T04:34:10.894168+0000 mon.a (mon.0) 871 : cluster 3 Health check update: Degraded data redundancy: 43998/262044 objects degraded (16.790%), 13 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:11.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[79733]: cluster 2023-10-09T04:34:10.894123+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:34:11.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[79733]: cluster 2023-10-09T04:34:10.894168+0000 mon.a (mon.0) 871 : cluster 3 Health check update: Degraded data redundancy: 43998/262044 objects degraded (16.790%), 13 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:11.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[79733]: audit 2023-10-09T04:34:11.339037+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.82:0/811866036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:11.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:11.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:12.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:12.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:12.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:12.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:12.173 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:11 smithi082 bash[77063]: audit 2023-10-09T04:34:11.339037+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.82:0/811866036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:11 smithi153 bash[59565]: cluster 2023-10-09T04:34:10.894123+0000 mon.a (mon.0) 870 : cluster 3 Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-10-09T04:34:12.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:11 smithi153 bash[59565]: cluster 2023-10-09T04:34:10.894168+0000 mon.a (mon.0) 871 : cluster 3 Health check update: Degraded data redundancy: 43998/262044 objects degraded (16.790%), 13 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:12.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:11 smithi153 bash[59565]: audit 2023-10-09T04:34:11.339037+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.82:0/811866036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:12.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:12.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:12.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:12.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:12.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:12.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:12.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:12.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:12.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:12.693 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:12.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:12.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:12.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:12.926 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:12.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:12.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:13.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:13.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:12 smithi153 bash[59565]: cluster 2023-10-09T04:34:12.815661+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v568: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 7 peering, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 43998/261453 objects degraded (16.828%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:13.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.201 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:12 smithi082 bash[77063]: cluster 2023-10-09T04:34:12.815661+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v568: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 7 peering, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 43998/261453 objects degraded (16.828%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:13.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:12 smithi082 bash[79733]: cluster 2023-10-09T04:34:12.815661+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v568: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 7 peering, 48 active+clean, 3 undersized+peered, 17 active+undersized, 3 undersized+degraded+peered, 9 active+undersized+degraded; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 43998/261453 objects degraded (16.828%); 85 KiB/s, 1 objects/s recovering 2023-10-09T04:34:13.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:13.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:13.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:13.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:13.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:13.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:13.591 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:13.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:13.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:13.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:13.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:14.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:14.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:14.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:14.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:14.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:14.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:14.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:14.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:14.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:14.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:14.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:14.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:14.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:14.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:14.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:14.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:14.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:14.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:14.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:15.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:15.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:15.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:34:15.194 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:34:15.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:15.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:15.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:15.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[77063]: audit 2023-10-09T04:34:14.271633+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[77063]: audit 2023-10-09T04:34:14.272928+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[77063]: cluster 2023-10-09T04:34:14.816655+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v569: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 61 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 147 B/s rd, 138 KiB/s wr, 201 op/s; 26777/257937 objects degraded (10.381%); 193 KiB/s, 3 objects/s recovering 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[77063]: audit 2023-10-09T04:34:14.850755+0000 mon.a (mon.0) 874 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[79733]: audit 2023-10-09T04:34:14.271633+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[79733]: audit 2023-10-09T04:34:14.272928+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[79733]: cluster 2023-10-09T04:34:14.816655+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v569: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 61 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 147 B/s rd, 138 KiB/s wr, 201 op/s; 26777/257937 objects degraded (10.381%); 193 KiB/s, 3 objects/s recovering 2023-10-09T04:34:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:15 smithi082 bash[79733]: audit 2023-10-09T04:34:14.850755+0000 mon.a (mon.0) 874 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:15.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:15.531 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:144121 5.b 5:d1751e0f:::benchmark_data_smithi082_106363_object29890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:23.682107+0000", 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 291.845040305, 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.32086491, 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.532 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144121 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682107+0000", 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.533 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682107+0000", 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682105+0000", 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.534 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682112+0000", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1489999999999999e-06 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682124+0000", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2468e-05 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682132+0000", 2023-10-09T04:34:15.535 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3729999999999993e-06 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682348+0000", 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021531600000000001 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682691+0000", 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00034288800000000001 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.536 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.682808+0000", 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000117626 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.688738+0000", 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0059298270000000004 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.537 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.002824+0000", 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.31408554799999999 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.002879+0000", 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5089000000000001e-05 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.002972+0000", 2023-10-09T04:34:15.538 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2745999999999996e-05 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:144122 5.8 5:1008f120:::benchmark_data_smithi082_106363_object29891:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:23.687503+0000", 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 291.83964409100003, 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.318017353, 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.539 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144122 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687503+0000", 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.540 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687503+0000", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687501+0000", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687511+0000", 2023-10-09T04:34:15.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8430000000000005e-06 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687523+0000", 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2349e-05 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687530+0000", 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8530000000000003e-06 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687582+0000", 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1836999999999999e-05 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687717+0000", 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013464999999999999 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.687831+0000", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000114708 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.692482+0000", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0046511570000000004 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.692706+0000", 2023-10-09T04:34:15.544 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00022347 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.005133+0000", 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.31242688600000001 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.005247+0000", 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000114543 2023-10-09T04:34:15.545 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.005520+0000", 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027288099999999999 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:144131 5.b 5:d5dd1454:::benchmark_data_smithi082_106363_object29900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T04:34:15.546 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:23.713891+0000", 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 291.81325572499998, 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.29233680899999998, 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144131 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.547 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713891+0000", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713891+0000", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713890+0000", 2023-10-09T04:34:15.548 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713898+0000", 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7870000000000002e-06 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713910+0000", 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2756e-05 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.549 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713919+0000", 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3979999999999999e-06 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.713980+0000", 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0822999999999997e-05 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.550 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.714103+0000", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012344400000000001 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.714206+0000", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000103379 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:23.718169+0000", 2023-10-09T04:34:15.551 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039626269999999998 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.006096+0000", 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28792674299999998 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.006143+0000", 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7209000000000002e-05 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.552 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:24.006228+0000", 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5029000000000002e-05 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:147914 5.8 5:1af9a23c:::benchmark_data_smithi082_106363_object33683:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:29:28.879075+0000", 2023-10-09T04:34:15.553 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 286.64807149400002, 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20502213399999999, 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 147914 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.554 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879075+0000", 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879075+0000", 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879074+0000", 2023-10-09T04:34:15.555 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879081+0000", 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8059999999999997e-06 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879093+0000", 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2109e-05 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.556 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879099+0000", 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7559999999999996e-06 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879132+0000", 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3358999999999997e-05 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.557 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879217+0000", 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4770000000000003e-05 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.879284+0000", 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6896000000000005e-05 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.884012+0000", 2023-10-09T04:34:15.558 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0047282349999999999 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:28.885233+0000", 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0012208589999999999 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:29.084007+0000", 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.198774386 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.559 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:29.084057+0000", 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9490000000000002e-05 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:29:29.084098+0000", 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0995000000000003e-05 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.560 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:211302 5.b 5:d5612dec:::benchmark_data_smithi082_106363_object97071:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:30:55.345738+0000", 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 200.18140851199999, 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.30215699499999998, 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 211302 2023-10-09T04:34:15.561 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345738+0000", 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345738+0000", 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345737+0000", 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345743+0000", 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6679999999999998e-06 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345753+0000", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7510000000000003e-06 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.345760+0000", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.365124+0000", 2023-10-09T04:34:15.564 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019364183 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.365267+0000", 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014221899999999999 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.365391+0000", 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012475 2023-10-09T04:34:15.565 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.371931+0000", 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065392009999999997 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.647770+0000", 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.27583955599999999 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.566 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.647810+0000", 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0144000000000001e-05 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.647895+0000", 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5153000000000003e-05 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.567 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:211305 5.8 5:1cef2dfc:::benchmark_data_smithi082_106363_object97074:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:30:55.356644+0000", 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 200.170502505, 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28209626999999998, 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:34:15.568 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 211305 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356644+0000", 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356644+0000", 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.569 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356644+0000", 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356648+0000", 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.138e-06 2023-10-09T04:34:15.570 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356657+0000", 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7099999999999996e-06 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356663+0000", 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6449999999999999e-06 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.571 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356703+0000", 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9644000000000002e-05 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.356865+0000", 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016162199999999999 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:34:15.572 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.357074+0000", 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000209514 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.362703+0000", 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056289310000000002 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.367998+0000", 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0052945479999999996 2023-10-09T04:34:15.573 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.638648+0000", 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.27065062000000001 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.638699+0000", 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0430000000000003e-05 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.574 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:30:55.638741+0000", 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2067999999999998e-05 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22571 6.1 e136/132 6:82bac691:::benchmark_data_smithi082_128868_object22570:head v 136'1406, mlcod=136'1404)", 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.021293+0000", 2023-10-09T04:34:15.575 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.505854169, 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23255816000000001, 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021293+0000", 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.576 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021293+0000", 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021292+0000", 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021317+0000", 2023-10-09T04:34:15.577 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5147000000000002e-05 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021326+0000", 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2059999999999996e-06 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021341+0000", 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5084e-05 2023-10-09T04:34:15.578 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021385+0000", 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3735000000000001e-05 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021407+0000", 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1537000000000001e-05 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.579 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024796+0000", 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0033891139999999999 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253851+0000", 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22905514699999999 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.580 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:22578 6.2 6:433d4c33:::benchmark_data_smithi082_128868_object22577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.021601+0000", 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.505546009, 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.24359618, 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.581 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 22578 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021601+0000", 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021601+0000", 2023-10-09T04:34:15.582 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021599+0000", 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021629+0000", 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0103999999999998e-05 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.583 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021642+0000", 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3259e-05 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021650+0000", 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9619999999999993e-06 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.584 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021732+0000", 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2226999999999997e-05 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021870+0000", 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013717100000000001 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:34:15.585 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023602+0000", 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001732602 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.262524+0000", 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.238921315 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.262564+0000", 2023-10-09T04:34:15.586 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0358000000000002e-05 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265105+0000", 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0025412989999999999 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265156+0000", 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1311999999999999e-05 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.587 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265197+0000", 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0602999999999997e-05 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22573 6.e e136/132 6:793e1913:::benchmark_data_smithi082_128868_object22572:head v 136'1351, mlcod=136'1349)", 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.021953+0000", 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.50519416900001, 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23515670599999999, 2023-10-09T04:34:15.588 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021953+0000", 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021953+0000", 2023-10-09T04:34:15.589 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.021952+0000", 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.022018+0000", 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6285000000000007e-05 2023-10-09T04:34:15.590 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.022028+0000", 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4630000000000007e-06 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.022034+0000", 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4690000000000002e-06 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.591 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.022098+0000", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4152000000000005e-05 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.022555+0000", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00045646599999999999 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.257028+0000", 2023-10-09T04:34:15.592 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23447346499999999 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.257109+0000", 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1243999999999997e-05 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:22580 6.2 6:4767f2c2:::benchmark_data_smithi082_128868_object22579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:34:15.593 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.023492+0000", 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.503655354, 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.24180189899999999, 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 22580 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.594 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023492+0000", 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023492+0000", 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.595 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023490+0000", 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023516+0000", 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6486e-05 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023531+0000", 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4742e-05 2023-10-09T04:34:15.596 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023540+0000", 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2960000000000004e-06 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024585+0000", 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001044861 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.597 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253896+0000", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22931089099999999 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.254147+0000", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00025091300000000001 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.262593+0000", 2023-10-09T04:34:15.598 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0084460019999999993 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.262624+0000", 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1121999999999998e-05 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265208+0000", 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0025837799999999999 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.599 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265259+0000", 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1029999999999998e-05 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.265293+0000", 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4536999999999998e-05 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.600 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22579 6.8 e136/132 6:189a18fb:::benchmark_data_smithi082_128868_object22578:head v 136'1408, mlcod=136'1406)", 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.023934+0000", 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.503213336, 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.229749918, 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023934+0000", 2023-10-09T04:34:15.601 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023934+0000", 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.023933+0000", 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.602 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024068+0000", 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000135372 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024080+0000", 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2388999999999999e-05 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.603 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024088+0000", 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9389999999999997e-06 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024143+0000", 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4351000000000002e-05 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024163+0000", 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0049000000000001e-05 2023-10-09T04:34:15.604 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253564+0000", 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22940137299999999 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253683+0000", 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011948899999999999 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.605 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22581 6.e e136/132 6:754d3d1d:::benchmark_data_smithi082_128868_object22580:head v 136'1352, mlcod=136'1349)", 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.024707+0000", 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.50243989400001, 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23430673900000001, 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.606 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024707+0000", 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024707+0000", 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024706+0000", 2023-10-09T04:34:15.607 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024768+0000", 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1403999999999997e-05 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024779+0000", 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1844999999999999e-05 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.608 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.024787+0000", 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.481e-06 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253607+0000", 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.228820198 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.609 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253701+0000", 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4247999999999996e-05 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.258939+0000", 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0052371830000000003 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.259014+0000", 2023-10-09T04:34:15.610 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5202000000000003e-05 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:22586 6.f 6:fc32bf93:::benchmark_data_smithi082_128868_object22585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.025965+0000", 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.50118156000001, 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23507383500000001, 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.611 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 22586 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.025965+0000", 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.612 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.025965+0000", 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.025963+0000", 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.613 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.025992+0000", 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8670000000000002e-05 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253623+0000", 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22763137 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.614 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253654+0000", 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1031999999999997e-05 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253788+0000", 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013324499999999999 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.254102+0000", 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031419100000000001 2023-10-09T04:34:15.615 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.254415+0000", 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031337699999999999 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.260282+0000", 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0058665430000000001 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.616 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.260301+0000", 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9748000000000001e-05 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.260472+0000", 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000170948 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.617 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.260549+0000", 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6507000000000005e-05 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.261039+0000", 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000490242 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.618 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22589 6.e e136/132 6:78dafd59:::benchmark_data_smithi082_128868_object22588:head v 136'1353, mlcod=136'1349)", 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.027483+0000", 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.499663879, 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23440847000000001, 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.027483+0000", 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.619 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.027483+0000", 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.027482+0000", 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.620 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.027548+0000", 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5913000000000004e-05 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253491+0000", 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22594354 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.621 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253525+0000", 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3762e-05 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.255460+0000", 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019347629999999999 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.255503+0000", 2023-10-09T04:34:15.622 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3590000000000001e-05 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.261644+0000", 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0061400659999999996 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.261891+0000", 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00024796399999999999 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.623 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:22595 6.9 e136/132 6:9bec12be:::benchmark_data_smithi082_128868_object22594:head v 136'1435, mlcod=136'1433)", 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:19.030432+0000", 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 116.49671518300001, 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22841004500000001, 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.624 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.030432+0000", 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.030432+0000", 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.625 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.030431+0000", 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.030463+0000", 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.2668e-05 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.626 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253532+0000", 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.223068289 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253564+0000", 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.2122000000000002e-05 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253623+0000", 2023-10-09T04:34:15.627 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8996000000000002e-05 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.253650+0000", 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.779e-05 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.257910+0000", 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0042591950000000003 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.628 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:19.258842+0000", 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00093212500000000001 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87199 6.2 6:44a33b73:::benchmark_data_smithi082_128868_object87198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.723166+0000", 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.803980920000001, 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.036921894, 2023-10-09T04:34:15.629 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87199 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723166+0000", 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.630 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723166+0000", 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723164+0000", 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.631 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723188+0000", 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3193e-05 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723204+0000", 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6813999999999999e-05 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.632 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723212+0000", 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0979999999999994e-06 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723249+0000", 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6779999999999997e-05 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723379+0000", 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012999100000000001 2023-10-09T04:34:15.633 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.723519+0000", 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013942699999999999 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.727440+0000", 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039213520000000003 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.634 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.729217+0000", 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0017767659999999999 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.751637+0000", 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.022419822999999998 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.635 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.751648+0000", 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1749000000000001e-05 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.760008+0000", 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.008359137 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.636 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.760088+0000", 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0365999999999996e-05 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87219 6.2 6:4ffa68c3:::benchmark_data_smithi082_128868_object87218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.756112+0000", 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.771035057000006, 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0180214999999999, 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.637 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87219 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756112+0000", 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.638 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756112+0000", 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756110+0000", 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.639 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756140+0000", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0198000000000001e-05 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756153+0000", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3072e-05 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756162+0000", 2023-10-09T04:34:15.640 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1039999999999996e-06 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763111+0000", 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0069489369999999996 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763124+0000", 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2537e-05 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.641 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.760384+0000", 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.99726032499999995 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.760543+0000", 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000158942 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:15.642 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.760714+0000", 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000171037 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.773987+0000", 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013273167000000001 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.774039+0000", 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1415000000000002e-05 2023-10-09T04:34:15.643 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.774072+0000", 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3231000000000001e-05 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.774133+0000", 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1259000000000004e-05 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.644 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87220 6.2 6:4547bb46:::benchmark_data_smithi082_128868_object87219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.756170+0000", 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.770977259000006, 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.020767588, 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.645 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87220 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756170+0000", 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.646 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756170+0000", 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756169+0000", 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756194+0000", 2023-10-09T04:34:15.647 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5154e-05 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756202+0000", 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4089999999999997e-06 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.756206+0000", 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1960000000000001e-06 2023-10-09T04:34:15.648 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763287+0000", 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0070812929999999998 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763297+0000", 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.9329999999999996e-06 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.649 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.763420+0000", 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.000122612 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.763605+0000", 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000185265 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:15.650 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.763772+0000", 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016723199999999999 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.773996+0000", 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010224024999999999 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.776850+0000", 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002853834 2023-10-09T04:34:15.651 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.776883+0000", 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3389000000000002e-05 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.776937+0000", 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3779999999999997e-05 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.652 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87229 6.2 6:4078ec4a:::benchmark_data_smithi082_128868_object87228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.763854+0000", 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.763292562999993, 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0133565390000001, 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.653 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87229 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763854+0000", 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763854+0000", 2023-10-09T04:34:15.654 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.655 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763853+0000", 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763873+0000", 2023-10-09T04:34:15.656 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9814e-05 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763885+0000", 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2524000000000001e-05 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763894+0000", 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3869999999999995e-06 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.657 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763930+0000", 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6273000000000003e-05 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.763935+0000", 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0590000000000002e-06 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.658 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.766734+0000", 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0027986310000001 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.766900+0000", 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016619700000000001 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.767088+0000", 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018759399999999999 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.774002+0000", 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0069147080000000003 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.776979+0000", 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0029770930000000001 2023-10-09T04:34:15.660 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.777021+0000", 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1838000000000002e-05 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.777211+0000", 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018969599999999999 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.661 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87253 6.2 6:488ceadb:::benchmark_data_smithi082_128868_object87252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.805609+0000", 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 65.721538096000003, 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.97484897800000003, 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:15.662 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87253 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805609+0000", 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805609+0000", 2023-10-09T04:34:15.663 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805607+0000", 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805632+0000", 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5153000000000001e-05 2023-10-09T04:34:15.664 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805648+0000", 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6172e-05 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805657+0000", 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9639999999999992e-06 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.665 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805713+0000", 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6023000000000002e-05 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.805722+0000", 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9469999999999992e-06 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.666 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.770772+0000", 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96504961600000005 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.770900+0000", 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000128319 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:15.667 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.771043+0000", 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000142955 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.774148+0000", 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0031050930000000002 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.780327+0000", 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0061784709999999996 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.668 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.780381+0000", 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4589000000000002e-05 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.780458+0000", 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6412e-05 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.669 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.670 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:15.670 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:15.670 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:34:15.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:15 smithi153 bash[59565]: audit 2023-10-09T04:34:14.271633+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:15 smithi153 bash[59565]: audit 2023-10-09T04:34:14.272928+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:15 smithi153 bash[59565]: cluster 2023-10-09T04:34:14.816655+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v569: 89 pgs: 1 active+clean+remapped, 1 active+recovering+degraded, 61 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 147 B/s rd, 138 KiB/s wr, 201 op/s; 26777/257937 objects degraded (10.381%); 193 KiB/s, 3 objects/s recovering 2023-10-09T04:34:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:15 smithi153 bash[59565]: audit 2023-10-09T04:34:14.850755+0000 mon.a (mon.0) 874 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:15.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:15.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:15.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:15.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:15.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:16.060 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:16.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:16.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:16.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:16.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:16.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.462 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:16 smithi153 bash[59565]: cluster 2023-10-09T04:34:15.568802+0000 mon.a (mon.0) 875 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive) 2023-10-09T04:34:16.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:16.525 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:34:16.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:16.591 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:16 smithi082 bash[77063]: cluster 2023-10-09T04:34:15.568802+0000 mon.a (mon.0) 875 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive) 2023-10-09T04:34:16.591 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:16 smithi082 bash[79733]: cluster 2023-10-09T04:34:15.568802+0000 mon.a (mon.0) 875 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive) 2023-10-09T04:34:16.614 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:16.723 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:16.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:16.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:16.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:16.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:17.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:17.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:17.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:17.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:34:17.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:17.283 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:17.390 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:17.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:17.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:17 smithi153 bash[59565]: cluster 2023-10-09T04:34:16.817527+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v570: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 120 KiB/s wr, 406 op/s; 26491/252381 objects degraded (10.496%); 184 KiB/s, 2 objects/s recovering 2023-10-09T04:34:17.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:17 smithi082 bash[77063]: cluster 2023-10-09T04:34:16.817527+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v570: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 120 KiB/s wr, 406 op/s; 26491/252381 objects degraded (10.496%); 184 KiB/s, 2 objects/s recovering 2023-10-09T04:34:17.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:17 smithi082 bash[79733]: cluster 2023-10-09T04:34:16.817527+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v570: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 120 KiB/s wr, 406 op/s; 26491/252381 objects degraded (10.496%); 184 KiB/s, 2 objects/s recovering 2023-10-09T04:34:17.519 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:34:17.523 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:17.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:17.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:17.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:17.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:17.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:17.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:17.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.003 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:18.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:18.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:18.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:18.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:18.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:18.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:18 smithi082 bash[77063]: cluster 2023-10-09T04:34:17.273203+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 26491/252381 objects degraded (10.496%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:18.396 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:18 smithi082 bash[79733]: cluster 2023-10-09T04:34:17.273203+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 26491/252381 objects degraded (10.496%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:18.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:18.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:18.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:18.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.670 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:18 smithi153 bash[59565]: cluster 2023-10-09T04:34:17.273203+0000 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 26491/252381 objects degraded (10.496%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:18.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:18.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:18.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:18.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:18.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:18.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:19.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:19.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:19.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:19.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:19.320 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:34:19.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:19.352 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:19 smithi153 bash[59565]: audit 2023-10-09T04:34:18.397591+0000 mon.a (mon.0) 877 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:19.353 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:19 smithi153 bash[59565]: cluster 2023-10-09T04:34:18.818503+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v571: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 103 B/s rd, 97 KiB/s wr, 519 op/s; 25807/246663 objects degraded (10.462%); 148 KiB/s, 2 objects/s recovering 2023-10-09T04:34:19.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:19.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:19.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:19 smithi082 bash[77063]: audit 2023-10-09T04:34:18.397591+0000 mon.a (mon.0) 877 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:19.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:19 smithi082 bash[77063]: cluster 2023-10-09T04:34:18.818503+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v571: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 103 B/s rd, 97 KiB/s wr, 519 op/s; 25807/246663 objects degraded (10.462%); 148 KiB/s, 2 objects/s recovering 2023-10-09T04:34:19.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:19 smithi082 bash[79733]: audit 2023-10-09T04:34:18.397591+0000 mon.a (mon.0) 877 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:19.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:19 smithi082 bash[79733]: cluster 2023-10-09T04:34:18.818503+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v571: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 103 B/s rd, 97 KiB/s wr, 519 op/s; 25807/246663 objects degraded (10.462%); 148 KiB/s, 2 objects/s recovering 2023-10-09T04:34:19.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:19.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:19.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:19.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:19.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:19.755 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:19] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:34:19.782 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:19.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:19.891 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:19.914 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:19.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:20.017 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:20.057 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:20.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:20.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.294279+0000 mon.a (mon.0) 878 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.300717+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.300933+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: audit 2023-10-09T04:34:19.304533+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.304582+0000 mon.a (mon.0) 880 : cluster 0 osdmap e148: 8 total, 7 up, 8 in 2023-10-09T04:34:20.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.306985+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.294279+0000 mon.a (mon.0) 878 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.300717+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.300933+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: audit 2023-10-09T04:34:19.304533+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.304582+0000 mon.a (mon.0) 880 : cluster 0 osdmap e148: 8 total, 7 up, 8 in 2023-10-09T04:34:20.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:20 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.306985+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:20.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.294279+0000 mon.a (mon.0) 878 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.300717+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.300933+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: audit 2023-10-09T04:34:19.304533+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.82:0/1866826206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.304582+0000 mon.a (mon.0) 880 : cluster 0 osdmap e148: 8 total, 7 up, 8 in 2023-10-09T04:34:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:20 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.306985+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:20.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:20.818 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:20.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:20.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:20.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:21.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:21.115 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:21.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:21.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:21.322 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:34:21.322 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:34:21.322 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.322 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:97261 6.c 6:3930dfbf:::benchmark_data_smithi082_128868_object9923:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:34:21.322 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:21.305455+0000", 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.01477007, 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014813288000000001, 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 97261 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.323 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305455+0000", 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305455+0000", 2023-10-09T04:34:21.324 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305453+0000", 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0032e-05 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:21.325 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305463+0000", 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9998999999999999e-05 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305483+0000", 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6829999999999997e-06 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.326 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305492+0000", 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000189459 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305681+0000", 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015194199999999999 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:21.327 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305833+0000", 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015757100000000001 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305991+0000", 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013099341 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:34:21.328 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.319090+0000", 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013635156000000001 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:97262 6.c 6:3a1bc041:::benchmark_data_smithi082_128868_object9924:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:21.305508+0000", 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.014717188000000001, 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014880015999999999, 2023-10-09T04:34:21.329 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 97262 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:21.330 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305508+0000", 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305508+0000", 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.331 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305507+0000", 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2760000000000002e-06 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305511+0000", 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5750000000000003e-06 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.332 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305519+0000", 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.724e-06 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305524+0000", 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044515739999999998 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.333 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.309975+0000", 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00037808399999999999 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.310353+0000", 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00050832600000000003 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.334 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.310862+0000", 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0082623409999999994 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.319124+0000", 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013616224 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.335 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:97264 6.c 6:33f45685:::benchmark_data_smithi082_128868_object9926:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:21.305529+0000", 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.014696252999999999, 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.01497941, 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:34:21.336 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 97264 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305529+0000", 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.337 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305529+0000", 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305528+0000", 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0089999999999999e-06 2023-10-09T04:34:21.338 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305531+0000", 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9789999999999998e-06 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305537+0000", 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.048e-06 2023-10-09T04:34:21.339 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.305541+0000", 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0091577689999999996 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.314699+0000", 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00039369300000000003 2023-10-09T04:34:21.340 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.315093+0000", 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00052297499999999996 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.315616+0000", 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00353258 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.341 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.319148+0000", 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013619584000000001 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.342 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:97274 6.3 e148/146 6:ce75e37d:::benchmark_data_smithi082_128868_object9936:head v 148'5970, mlcod=148'5968)", 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:21.320447+0000", 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 4294967295.9997778, 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014501800000000001, 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320447+0000", 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:21.343 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320447+0000", 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320446+0000", 2023-10-09T04:34:21.344 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3170000000000008e-06 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320455+0000", 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3200000000000001e-05 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320468+0000", 2023-10-09T04:34:21.345 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.81e-06 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320475+0000", 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3596e-05 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:21.346 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320529+0000", 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9580000000000001e-05 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:21.320558+0000", 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011134899999999999 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:21.347 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.348 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:21.348 INFO:teuthology.orchestra.run.smithi153.stdout: ], 2023-10-09T04:34:21.348 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 4 2023-10-09T04:34:21.348 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:34:21.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.306145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[77063]: cluster 2023-10-09T04:34:19.307271+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[77063]: cluster 2023-10-09T04:34:20.819787+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v573: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 96 KiB/s wr, 807 op/s; 24885/237900 objects degraded (10.460%); 96 KiB/s, 1 objects/s recovering 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.306145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[79733]: cluster 2023-10-09T04:34:19.307271+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:21 smithi082 bash[79733]: cluster 2023-10-09T04:34:20.819787+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v573: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 96 KiB/s wr, 807 op/s; 24885/237900 objects degraded (10.460%); 96 KiB/s, 1 objects/s recovering 2023-10-09T04:34:21.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:21.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:21 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.306145+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:21 smithi153 bash[59565]: cluster 2023-10-09T04:34:19.307271+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T04:34:21.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:21 smithi153 bash[59565]: cluster 2023-10-09T04:34:20.819787+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v573: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 96 KiB/s wr, 807 op/s; 24885/237900 objects degraded (10.460%); 96 KiB/s, 1 objects/s recovering 2023-10-09T04:34:21.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:21.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:21.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:21.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:21.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:21.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:21.885 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:21.889 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:21.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:21.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:21.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:22.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:22.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:22.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:22.186 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:22.251 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-10-09T04:34:22.251 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-10-09T04:34:22.251 DEBUG:teuthology.orchestra.run.smithi153:osd.5> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T04:34:22.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl start ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T04:34:22.306 INFO:journalctl@ceph.osd.5.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:34:22.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:22.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:34:22.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:22.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:22.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:22.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:22.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.628 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:22 smithi153 bash[59565]: audit 2023-10-09T04:34:21.330584+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.628 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:22 smithi153 bash[59565]: audit 2023-10-09T04:34:21.340509+0000 mon.a (mon.0) 882 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.629 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:22 smithi153 bash[59565]: audit 2023-10-09T04:34:21.736085+0000 mon.a (mon.0) 883 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.629 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:22 smithi153 bash[59565]: audit 2023-10-09T04:34:21.743055+0000 mon.a (mon.0) 884 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.629 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:22 smithi153 systemd[1]: Started Ceph osd.5 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:34:22.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:22.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[77063]: audit 2023-10-09T04:34:21.330584+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[77063]: audit 2023-10-09T04:34:21.340509+0000 mon.a (mon.0) 882 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[77063]: audit 2023-10-09T04:34:21.736085+0000 mon.a (mon.0) 883 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[77063]: audit 2023-10-09T04:34:21.743055+0000 mon.a (mon.0) 884 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[79733]: audit 2023-10-09T04:34:21.330584+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[79733]: audit 2023-10-09T04:34:21.340509+0000 mon.a (mon.0) 882 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[79733]: audit 2023-10-09T04:34:21.736085+0000 mon.a (mon.0) 883 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:22 smithi082 bash[79733]: audit 2023-10-09T04:34:21.743055+0000 mon.a (mon.0) 884 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:22.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:22.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:22.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:34:22.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:22.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:23.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.037 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:23.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:23.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:23.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:23.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:23.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:23.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:23.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:23.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[77063]: audit 2023-10-09T04:34:22.332105+0000 mon.a (mon.0) 885 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[77063]: audit 2023-10-09T04:34:22.333751+0000 mon.a (mon.0) 886 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[77063]: cluster 2023-10-09T04:34:22.336480+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v574: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 107 B/s rd, 101 KiB/s wr, 828 op/s; 24885/237900 objects degraded (10.460%); 101 KiB/s, 1 objects/s recovering 2023-10-09T04:34:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[77063]: audit 2023-10-09T04:34:22.342021+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[79733]: audit 2023-10-09T04:34:22.332105+0000 mon.a (mon.0) 885 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[79733]: audit 2023-10-09T04:34:22.333751+0000 mon.a (mon.0) 886 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[79733]: cluster 2023-10-09T04:34:22.336480+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v574: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 107 B/s rd, 101 KiB/s wr, 828 op/s; 24885/237900 objects degraded (10.460%); 101 KiB/s, 1 objects/s recovering 2023-10-09T04:34:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:23 smithi082 bash[79733]: audit 2023-10-09T04:34:22.342021+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:23.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.550 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:23.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:23 smithi153 bash[59565]: audit 2023-10-09T04:34:22.332105+0000 mon.a (mon.0) 885 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:23.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:23 smithi153 bash[59565]: audit 2023-10-09T04:34:22.333751+0000 mon.a (mon.0) 886 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:23.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:23 smithi153 bash[59565]: cluster 2023-10-09T04:34:22.336480+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v574: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 107 B/s rd, 101 KiB/s wr, 828 op/s; 24885/237900 objects degraded (10.460%); 101 KiB/s, 1 objects/s recovering 2023-10-09T04:34:23.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:23 smithi153 bash[59565]: audit 2023-10-09T04:34:22.342021+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:23.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.680 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:23.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:23.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:23.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:23.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:24.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:24.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:24.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:24.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:24.335 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:24.340 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:24 smithi082 bash[77063]: cluster 2023-10-09T04:34:23.328108+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-10-09T04:34:24.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:24.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:24.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:24.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:24.642 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:24 smithi082 bash[79733]: cluster 2023-10-09T04:34:23.328108+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-10-09T04:34:24.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:24.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[59565]: cluster 2023-10-09T04:34:23.328108+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-09T04:34:24.676 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:24 smithi153 bash[83333]: --> ceph-volume raw activate successful for osd ID: 5 2023-10-09T04:34:24.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:24.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:24.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:24.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:24.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:25.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:25.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:25.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:25.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:25.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:25.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:25.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:25 smithi082 bash[77063]: cluster 2023-10-09T04:34:24.337308+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v575: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 24319/232830 objects degraded (10.445%); 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:25.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:25 smithi082 bash[77063]: cluster 2023-10-09T04:34:24.388572+0000 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 24319/232830 objects degraded (10.445%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:25.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:25.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:25 smithi153 bash[59565]: cluster 2023-10-09T04:34:24.337308+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v575: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 24319/232830 objects degraded (10.445%); 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:25 smithi153 bash[59565]: cluster 2023-10-09T04:34:24.388572+0000 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 24319/232830 objects degraded (10.445%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:25.524 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:25.550 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:25.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:25.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:25.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:25.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:25 smithi082 bash[79733]: cluster 2023-10-09T04:34:24.337308+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v575: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 24319/232830 objects degraded (10.445%); 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:25.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:25 smithi082 bash[79733]: cluster 2023-10-09T04:34:24.388572+0000 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 24319/232830 objects degraded (10.445%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:25.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:25.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:25.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:25.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:25.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:26.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:26.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:26.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:26.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:26.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:26.339 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:26.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:26.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:26.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:26.611 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:26.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:26.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:26.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:26.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:26.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:26.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:27.011 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:27.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:27.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:27.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:27.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:27.350 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:27 smithi153 bash[83720]: debug 2023-10-09T04:34:27.095+0000 7f3750a03200 -1 Falling back to public interface 2023-10-09T04:34:27.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:27 smithi082 bash[77063]: cluster 2023-10-09T04:34:26.339436+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v576: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 23517/224781 objects degraded (10.462%) 2023-10-09T04:34:27.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:27.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:27.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:27.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:27.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:27.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:27.640 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:27 smithi082 bash[79733]: cluster 2023-10-09T04:34:26.339436+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v576: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 23517/224781 objects degraded (10.462%) 2023-10-09T04:34:27.641 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:27 smithi153 bash[59565]: cluster 2023-10-09T04:34:26.339436+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v576: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 23517/224781 objects degraded (10.462%) 2023-10-09T04:34:27.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:27.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:27.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:27.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:27.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:27.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:28.044 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:28.073 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:28.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:28.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:28.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:28.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:28.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:28.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:28.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:28.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:28.815 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:34:28.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.875 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:28.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:28.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.003 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:29.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.123 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104238 6.f e148/132 6:f2b7210f:::benchmark_data_smithi082_128868_object16900:head v 148'6453, mlcod=148'6451)", 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.113262+0000", 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.011830830000000001, 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011871678, 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.127 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113262+0000", 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113262+0000", 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113261+0000", 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113269+0000", 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.542e-06 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113278+0000", 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3130000000000004e-06 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.113285+0000", 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2759999999999997e-06 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.117090+0000", 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0038057439999999998 2023-10-09T04:34:29.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.117174+0000", 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3639000000000003e-05 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104246 6.c e148/146 6:32b63807:::benchmark_data_smithi082_128868_object16908:head v 148'6579, mlcod=148'6576)", 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.114086+0000", 2023-10-09T04:34:29.131 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.011006884, 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011124408000000001, 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114086+0000", 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114086+0000", 2023-10-09T04:34:29.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114085+0000", 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114092+0000", 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3570000000000003e-06 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114102+0000", 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.9140000000000003e-06 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.114108+0000", 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3489999999999997e-06 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:29.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.117347+0000", 2023-10-09T04:34:29.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0032386559999999999 2023-10-09T04:34:29.135 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.135 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.120568+0000", 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0032210979999999999 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104239 6.2 e148/146 6:4439f705:::benchmark_data_smithi082_128868_object16901:head v 148'6518, mlcod=148'6516)", 2023-10-09T04:34:29.136 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.118793+0000", 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0063004080000000004, 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064933300000000003, 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118793+0000", 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.137 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118793+0000", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118791+0000", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118801+0000", 2023-10-09T04:34:29.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7699999999999996e-06 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118816+0000", 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4541e-05 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118824+0000", 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.636e-06 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118937+0000", 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011241199999999999 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118959+0000", 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2150999999999998e-05 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.140 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104245 6.2 e148/146)", 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.118842+0000", 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0062510969999999997, 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065112649999999996, 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118842+0000", 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.141 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118842+0000", 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118841+0000", 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118847+0000", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3709999999999999e-06 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118854+0000", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4320000000000002e-06 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.118859+0000", 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1000000000000003e-06 2023-10-09T04:34:29.143 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104248 6.5 e148/146 6:a721fd92:::benchmark_data_smithi082_128868_object16910:head v 148'6445, mlcod=148'6443)", 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.121571+0000", 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0035220130000000001, 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003835682, 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121571+0000", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121571+0000", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121570+0000", 2023-10-09T04:34:29.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121581+0000", 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1442e-05 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121596+0000", 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4331999999999999e-05 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121603+0000", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3919999999999997e-06 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.125134+0000", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003530362 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.125163+0000", 2023-10-09T04:34:29.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9556999999999998e-05 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:104249 6.b e148/132 6:d243e981:::benchmark_data_smithi082_128868_object16911:head v 148'6374, mlcod=148'6372)", 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:29.121617+0000", 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0034761549999999999, 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0038478259999999999, 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:34:29.148 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121617+0000", 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121617+0000", 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:34:29.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121616+0000", 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121620+0000", 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9140000000000001e-06 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121626+0000", 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4500000000000003e-06 2023-10-09T04:34:29.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121630+0000", 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1280000000000001e-06 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121668+0000", 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8154000000000001e-05 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:34:29.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:29.121688+0000", 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9805000000000001e-05 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 6 2023-10-09T04:34:29.152 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:34:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.153 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:29.176 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:34:29.220 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:34:29.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:29.284 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:34:29.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:29.315 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:34:29.375 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:34:29.401 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:29.419 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.420 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:34:29.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:29.439 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[77063]: cluster 2023-10-09T04:34:28.340136+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v577: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 22982/219714 objects degraded (10.460%) 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[77063]: audit 2023-10-09T04:34:29.081290+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.82:0/1442003507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[77063]: audit 2023-10-09T04:34:29.272330+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[77063]: audit 2023-10-09T04:34:29.273720+0000 mon.a (mon.0) 891 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[79733]: cluster 2023-10-09T04:34:28.340136+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v577: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 22982/219714 objects degraded (10.460%) 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[79733]: audit 2023-10-09T04:34:29.081290+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.82:0/1442003507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[79733]: audit 2023-10-09T04:34:29.272330+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:29.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[79733]: audit 2023-10-09T04:34:29.273720+0000 mon.a (mon.0) 891 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:29.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:29.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:29 smithi153 bash[59565]: cluster 2023-10-09T04:34:28.340136+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v577: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 22982/219714 objects degraded (10.460%) 2023-10-09T04:34:29.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:29 smithi153 bash[59565]: audit 2023-10-09T04:34:29.081290+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.82:0/1442003507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:29.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:29 smithi153 bash[59565]: audit 2023-10-09T04:34:29.272330+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:29.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:29 smithi153 bash[59565]: audit 2023-10-09T04:34:29.273720+0000 mon.a (mon.0) 891 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.658 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:29.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:29.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:29] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T04:34:29.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:29.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:29.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:29.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:29.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:30.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:30.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:30.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:30.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:30.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:30.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:30.542 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:30 smithi153 bash[59565]: cluster 2023-10-09T04:34:29.572566+0000 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 22982/219714 objects degraded (10.460%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:30.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:30.580 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:30 smithi082 bash[77063]: cluster 2023-10-09T04:34:29.572566+0000 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 22982/219714 objects degraded (10.460%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:30.580 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:30 smithi082 bash[79733]: cluster 2023-10-09T04:34:29.572566+0000 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 22982/219714 objects degraded (10.460%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:30.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:30.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:30.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:30.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:30.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:30.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:30.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:30.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:34:30.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:34:31.030 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:34:31.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:31.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:31.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:31.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:31.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:31.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:31.493 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.505 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:31.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:31.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:31.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:31.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:31.784 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:31 smithi082 bash[77063]: cluster 2023-10-09T04:34:30.340879+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v578: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:31.784 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:31 smithi082 bash[79733]: cluster 2023-10-09T04:34:30.340879+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v578: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:31.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:31.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.918 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:31 smithi153 bash[59565]: cluster 2023-10-09T04:34:30.340879+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v578: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:31.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:31.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:32.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:32.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:32.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:32.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:32.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:32.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:32.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:32.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:32.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.868 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:32.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:32.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:33.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:34:33.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:33.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:33.182 INFO:teuthology.orchestra.run.smithi153.stderr:no valid command found; 10 closest matches: 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:0 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:1 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:2 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:abort 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:assert 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs debug_inject_read_zeros 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs files list 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs stats 2023-10-09T04:34:33.183 INFO:teuthology.orchestra.run.smithi153.stderr:bluestore allocator dump block 2023-10-09T04:34:33.184 INFO:teuthology.orchestra.run.smithi153.stderr:bluestore allocator fragmentation block 2023-10-09T04:34:33.184 INFO:teuthology.orchestra.run.smithi153.stderr:admin_socket: invalid command 2023-10-09T04:34:33.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:33.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:33.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:33.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:33.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:33.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:33 smithi153 bash[59565]: cluster 2023-10-09T04:34:32.341804+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v579: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:33.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:33.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:33.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:33 smithi082 bash[77063]: cluster 2023-10-09T04:34:32.341804+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v579: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:33.721 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:33 smithi082 bash[79733]: cluster 2023-10-09T04:34:32.341804+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v579: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 22178/211398 objects degraded (10.491%) 2023-10-09T04:34:33.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:33.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:33.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:33.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:34.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:34.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.228 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:34:34.229 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-10-09T04:34:34.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:34.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:34.374 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:34.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:34.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:34.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:34.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:34.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:34.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:34.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.773 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:34 smithi153 bash[59565]: audit 2023-10-09T04:34:34.529701+0000 mon.b (mon.1) 10 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.773 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:34 smithi153 bash[59565]: audit 2023-10-09T04:34:34.530830+0000 mon.a (mon.0) 893 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.773 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:34 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.574613+0000 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 22178/211398 objects degraded (10.491%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:34.774 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:34 smithi153 bash[83720]: debug 2023-10-09T04:34:34.522+0000 7f3750a03200 -1 osd.5 113 log_to_monitors {default=true} 2023-10-09T04:34:34.774 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 04:34:34 smithi153 bash[83720]: debug 2023-10-09T04:34:34.634+0000 7f3742eff700 -1 osd.5 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:34:34.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:34.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[77063]: audit 2023-10-09T04:34:34.529701+0000 mon.b (mon.1) 10 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[77063]: audit 2023-10-09T04:34:34.530830+0000 mon.a (mon.0) 893 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.574613+0000 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 22178/211398 objects degraded (10.491%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[79733]: audit 2023-10-09T04:34:34.529701+0000 mon.b (mon.1) 10 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[79733]: audit 2023-10-09T04:34:34.530830+0000 mon.a (mon.0) 893 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:34:34.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:34 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.574613+0000 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 22178/211398 objects degraded (10.491%), 12 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:34.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:34.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:35.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:35.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:35.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:35.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:35.280 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:35.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:35.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:35.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:35.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:35.581 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:35.582 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:35.686 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:35.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:35.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.342571+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v580: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 21587/206196 objects degraded (10.469%) 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.601073+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: audit 2023-10-09T04:34:34.611108+0000 mon.a (mon.0) 895 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.611249+0000 mon.a (mon.0) 896 : cluster 0 osdmap e149: 8 total, 7 up, 8 in 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.611457+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: audit 2023-10-09T04:34:34.612018+0000 mon.b (mon.1) 11 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: audit 2023-10-09T04:34:34.618423+0000 mon.a (mon.0) 897 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.618911+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.621198+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[79733]: cluster 2023-10-09T04:34:34.622676+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.342571+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v580: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 21587/206196 objects degraded (10.469%) 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.601073+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: audit 2023-10-09T04:34:34.611108+0000 mon.a (mon.0) 895 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.611249+0000 mon.a (mon.0) 896 : cluster 0 osdmap e149: 8 total, 7 up, 8 in 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.611457+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: audit 2023-10-09T04:34:34.612018+0000 mon.b (mon.1) 11 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: audit 2023-10-09T04:34:34.618423+0000 mon.a (mon.0) 897 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.618911+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.621198+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.824 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:35 smithi082 bash[77063]: cluster 2023-10-09T04:34:34.622676+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:35.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.342571+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v580: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 21587/206196 objects degraded (10.469%) 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.601073+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: audit 2023-10-09T04:34:34.611108+0000 mon.a (mon.0) 895 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.611249+0000 mon.a (mon.0) 896 : cluster 0 osdmap e149: 8 total, 7 up, 8 in 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.611457+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: audit 2023-10-09T04:34:34.612018+0000 mon.b (mon.1) 11 : audit 1 from='osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: audit 2023-10-09T04:34:34.618423+0000 mon.a (mon.0) 897 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.618911+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.621198+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:35 smithi153 bash[59565]: cluster 2023-10-09T04:34:34.622676+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T04:34:35.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:35.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:35.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:36.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:36.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:36.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:36.235 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:36.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:36.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:36.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:36.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:36.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:36.634 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:34:36.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:36.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:36.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:36.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.600976+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.620257+0000 mon.a (mon.0) 899 : cluster 1 osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594] boot 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.620486+0000 mon.a (mon.0) 900 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: audit 2023-10-09T04:34:35.625918+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.626372+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.626669+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.628281+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.629742+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.632285+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.634914+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.635577+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: audit 2023-10-09T04:34:36.035517+0000 mon.a (mon.0) 902 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.614731+0000 mon.a (mon.0) 903 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: audit 2023-10-09T04:34:36.620964+0000 mon.a (mon.0) 904 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.621148+0000 mon.a (mon.0) 905 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.600976+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.620257+0000 mon.a (mon.0) 899 : cluster 1 osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594] boot 2023-10-09T04:34:36.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.620486+0000 mon.a (mon.0) 900 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: audit 2023-10-09T04:34:35.625918+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.626372+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.626669+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.628281+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.629742+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.632285+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.634914+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.635577+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: audit 2023-10-09T04:34:36.035517+0000 mon.a (mon.0) 902 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:36.882 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.614731+0000 mon.a (mon.0) 903 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:36.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: audit 2023-10-09T04:34:36.620964+0000 mon.a (mon.0) 904 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:36.883 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:36 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.621148+0000 mon.a (mon.0) 905 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:34:36.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:36.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:36.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.600976+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.620257+0000 mon.a (mon.0) 899 : cluster 1 osd.5 [v2:172.21.15.153:6816/1614088594,v1:172.21.15.153:6817/1614088594] boot 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.620486+0000 mon.a (mon.0) 900 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: audit 2023-10-09T04:34:35.625918+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.626372+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.626669+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.628281+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.629742+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.632285+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.634914+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.635577+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: audit 2023-10-09T04:34:36.035517+0000 mon.a (mon.0) 902 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.614731+0000 mon.a (mon.0) 903 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: audit 2023-10-09T04:34:36.620964+0000 mon.a (mon.0) 904 : audit 1 from='client.? 172.21.15.82:0/3888383230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:36.928 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:36 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.621148+0000 mon.a (mon.0) 905 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:34:37.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:37.027 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:37.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:37.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:37.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:37.292 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:37.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:37.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:37.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:37.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:37.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:37.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.563446+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:34:37.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:35.615547+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:37.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.343459+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v583: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 20743/197763 objects degraded (10.489%) 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.618065+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.620719+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.620834+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.622579+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.622760+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.623878+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.625259+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.626334+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:36.626727+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[79733]: cluster 2023-10-09T04:34:37.626124+0000 mon.a (mon.0) 906 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.563446+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:35.615547+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.343459+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v583: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 20743/197763 objects degraded (10.489%) 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.618065+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.620719+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.620834+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.622579+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.622760+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.623878+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.625259+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.626334+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:36.626727+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.734 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:37 smithi082 bash[77063]: cluster 2023-10-09T04:34:37.626124+0000 mon.a (mon.0) 906 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:34:37.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:37.820 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:37.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.563446+0000 osd.5 (osd.5) 1 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T04:34:37.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:35.615547+0000 osd.5 (osd.5) 2 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T04:34:37.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.343459+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v583: 89 pgs: 1 active+clean+remapped, 62 active+clean, 14 active+undersized, 12 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 20743/197763 objects degraded (10.489%) 2023-10-09T04:34:37.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.618065+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.620719+0000 osd.5 (osd.5) 3 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.620834+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.622579+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.622760+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.623878+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.625259+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.626334+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:36.626727+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T04:34:37.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:37 smithi153 bash[59565]: cluster 2023-10-09T04:34:37.626124+0000 mon.a (mon.0) 906 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:34:37.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:37.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:38.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:38.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:38.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:38.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:38.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:38.441 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:38.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:38.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:38.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:38.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[77063]: cluster 2023-10-09T04:34:37.620839+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[77063]: cluster 2023-10-09T04:34:37.626642+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.740 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[77063]: cluster 2023-10-09T04:34:37.668248+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:38.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:38.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:38.906 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:38 smithi153 bash[59565]: cluster 2023-10-09T04:34:37.620839+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.906 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:38 smithi153 bash[59565]: cluster 2023-10-09T04:34:37.626642+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.906 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:38 smithi153 bash[59565]: cluster 2023-10-09T04:34:37.668248+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:38.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:38.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:39.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:39.046 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[79733]: cluster 2023-10-09T04:34:37.620839+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:39.046 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[79733]: cluster 2023-10-09T04:34:37.626642+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:39.046 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:38 smithi082 bash[79733]: cluster 2023-10-09T04:34:37.668248+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T04:34:39.079 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:39.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:39.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:39.230 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:34:39.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:39.364 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:39.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:39.492 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:39.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:39.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:39.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:39.763 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:39.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:39.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:39.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:39 smithi153 bash[59565]: cluster 2023-10-09T04:34:38.344416+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v586: 89 pgs: 3 activating+undersized+degraded+remapped, 69 active+clean, 10 active+undersized, 7 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 24687/194598 objects degraded (12.686%) 2023-10-09T04:34:39.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:39 smithi153 bash[59565]: cluster 2023-10-09T04:34:39.575690+0000 mon.a (mon.0) 907 : cluster 3 Health check update: Degraded data redundancy: 24687/194598 objects degraded (12.686%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:39.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:34:39.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:39 smithi082 bash[77063]: cluster 2023-10-09T04:34:38.344416+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v586: 89 pgs: 3 activating+undersized+degraded+remapped, 69 active+clean, 10 active+undersized, 7 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 24687/194598 objects degraded (12.686%) 2023-10-09T04:34:39.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:39 smithi082 bash[77063]: cluster 2023-10-09T04:34:39.575690+0000 mon.a (mon.0) 907 : cluster 3 Health check update: Degraded data redundancy: 24687/194598 objects degraded (12.686%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:39.980 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:39] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T04:34:39.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:39 smithi082 bash[79733]: cluster 2023-10-09T04:34:38.344416+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v586: 89 pgs: 3 activating+undersized+degraded+remapped, 69 active+clean, 10 active+undersized, 7 active+undersized+degraded; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 24687/194598 objects degraded (12.686%) 2023-10-09T04:34:39.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:39 smithi082 bash[79733]: cluster 2023-10-09T04:34:39.575690+0000 mon.a (mon.0) 907 : cluster 3 Health check update: Degraded data redundancy: 24687/194598 objects degraded (12.686%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:34:40.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:40.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:40.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:40.309 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:40.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:40.438 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:40.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:40.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:40.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:40.807 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:40.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:40.936 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:40.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:40.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:41.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:41.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:41.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:41.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:41.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:41.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:41.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:41.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:41.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:41.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:41.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:41.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:41.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:41.912 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:41.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:41.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:41 smithi153 bash[59565]: cluster 2023-10-09T04:34:40.345006+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v587: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:41.931 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:41 smithi082 bash[77063]: cluster 2023-10-09T04:34:40.345006+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v587: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:41.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:41 smithi082 bash[79733]: cluster 2023-10-09T04:34:40.345006+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v587: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:42.017 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:42.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:42.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:42.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:42.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:42.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:42.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:42.464 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:42.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:42.593 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:42.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:42.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:42.795 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:34:42.795 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:34:42.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:42.856 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:42.960 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:42.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:42.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:42.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:43.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:43.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:43.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:43.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:43.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:43.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:43.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:43.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:43.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:43.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.798 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:43 smithi153 bash[59565]: cluster 2023-10-09T04:34:42.345517+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v588: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:43.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:43.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:43.935 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:43 smithi082 bash[77063]: cluster 2023-10-09T04:34:42.345517+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v588: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:43.936 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:43 smithi082 bash[79733]: cluster 2023-10-09T04:34:42.345517+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v588: 89 pgs: 3 activating+undersized+degraded+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 27476/191199 objects degraded (14.370%) 2023-10-09T04:34:43.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:43.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:44.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:44.070 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:34:44.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:44.211 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:44.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:44.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:44.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:44.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:44.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:44.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:44.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:44.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:44.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:34:44.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:44.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:44.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:44.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:44.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:45.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.099 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:45.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:45.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:45.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:45.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:45.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[77063]: audit 2023-10-09T04:34:44.272205+0000 mon.a (mon.0) 908 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[77063]: audit 2023-10-09T04:34:44.273291+0000 mon.a (mon.0) 909 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[77063]: cluster 2023-10-09T04:34:44.347856+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v589: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 25877/186045 objects degraded (13.909%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:34:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[77063]: cluster 2023-10-09T04:34:44.576832+0000 mon.a (mon.0) 910 : cluster 3 Health check update: Degraded data redundancy: 27476/191199 objects degraded (14.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[79733]: audit 2023-10-09T04:34:44.272205+0000 mon.a (mon.0) 908 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[79733]: audit 2023-10-09T04:34:44.273291+0000 mon.a (mon.0) 909 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[79733]: cluster 2023-10-09T04:34:44.347856+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v589: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 25877/186045 objects degraded (13.909%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:34:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:45 smithi082 bash[79733]: cluster 2023-10-09T04:34:44.576832+0000 mon.a (mon.0) 910 : cluster 3 Health check update: Degraded data redundancy: 27476/191199 objects degraded (14.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:45.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:45.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:45 smithi153 bash[59565]: audit 2023-10-09T04:34:44.272205+0000 mon.a (mon.0) 908 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:45 smithi153 bash[59565]: audit 2023-10-09T04:34:44.273291+0000 mon.a (mon.0) 909 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:45 smithi153 bash[59565]: cluster 2023-10-09T04:34:44.347856+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v589: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 25877/186045 objects degraded (13.909%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:34:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:45 smithi153 bash[59565]: cluster 2023-10-09T04:34:44.576832+0000 mon.a (mon.0) 910 : cluster 3 Health check update: Degraded data redundancy: 27476/191199 objects degraded (14.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:45.745 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:45.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:45.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:45.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:46.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.053 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:46.157 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:46.184 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:46.211 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:46.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:46.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:46.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:46.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:46.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:46.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:46.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:46.818 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:46.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:46.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:47.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:47.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:47.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:47.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:47.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:47.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:47.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:47.656 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:47 smithi153 bash[59565]: cluster 2023-10-09T04:34:46.348542+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v590: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25877/180777 objects degraded (14.314%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:34:47.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:47 smithi082 bash[77063]: cluster 2023-10-09T04:34:46.348542+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v590: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25877/180777 objects degraded (14.314%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:34:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:47 smithi082 bash[79733]: cluster 2023-10-09T04:34:46.348542+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v590: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25877/180777 objects degraded (14.314%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:34:47.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:47.833 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:34:47.880 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:34:47.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:47.927 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:34:47.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:47.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:47.965 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:34:48.008 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:34:48.054 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:34:48.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:48.092 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:34:48.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:48.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:48.145 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:34:48.150 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.150 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:34:48.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:48.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:48.341 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:48.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:48.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:48.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:48 smithi082 bash[77063]: audit 2023-10-09T04:34:47.783038+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/860536390' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:48 smithi082 bash[79733]: audit 2023-10-09T04:34:47.783038+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/860536390' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:48.499 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:48.602 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:48.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:48 smithi153 bash[59565]: audit 2023-10-09T04:34:47.783038+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.82:0/860536390' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:48.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:48.772 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:48.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:48.876 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:48.912 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:48.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:48.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:49.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:49.053 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:49.090 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:34:49.090 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:34:49.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.090 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117877 6.4 e152/151 6:21741fcd:::benchmark_data_smithi082_128868_object30539:head v 152'7391, mlcod=136'1450)", 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.063071+0000", 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.023295415, 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.023320859999999999, 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063071+0000", 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063071+0000", 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063071+0000", 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5569999999999997e-06 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063075+0000", 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0920000000000003e-06 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063081+0000", 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5990000000000002e-06 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.093 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063085+0000", 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00050586499999999998 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063591+0000", 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2035e-05 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.063633+0000", 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00056170899999999997 2023-10-09T04:34:49.094 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117880 6.c e152/151 6:3a98186d:::benchmark_data_smithi082_128868_object30542:head v 152'7463, mlcod=136'1480)", 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.066136+0000", 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.020230305000000001, 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.020310147000000001, 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:49.095 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066136+0000", 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066136+0000", 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066135+0000", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2626000000000001e-05 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066147+0000", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3589e-05 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066161+0000", 2023-10-09T04:34:49.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9949999999999999e-06 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066168+0000", 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0531999999999995e-05 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066238+0000", 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.7446000000000001e-05 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066266+0000", 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012966599999999999 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117878 6.4 e152/151 6:24792ccd:::benchmark_data_smithi082_128868_object30540:head v 152'7392, mlcod=136'1450)", 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.066466+0000", 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.019901043, 2023-10-09T04:34:49.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.020027128000000002, 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066466+0000", 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066466+0000", 2023-10-09T04:34:49.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066465+0000", 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4510000000000003e-06 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066472+0000", 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0239e-05 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066482+0000", 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9300000000000002e-06 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.066487+0000", 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0056343030000000002 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:49.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.072122+0000", 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6023000000000002e-05 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.072178+0000", 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0057122149999999997 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.103 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117879 6.4 e152/151 6:2ca288a5:::benchmark_data_smithi082_128868_object30541:head v 152'7393, mlcod=136'1450)", 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.070083+0000", 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.016283688000000001, 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.016449852000000001, 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070083+0000", 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070083+0000", 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070082+0000", 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1291000000000001e-05 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070093+0000", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5338000000000001e-05 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070108+0000", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0929999999999997e-06 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070115+0000", 2023-10-09T04:34:49.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014779511 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084895+0000", 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1257999999999998e-05 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084926+0000", 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014843243000000001 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.107 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.108 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.108 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117882 6.c e152/151 6:3ba12289:::benchmark_data_smithi082_128868_object30544:head v 152'7464, mlcod=136'1480)", 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.070413+0000", 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.015953433999999999, 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.016160377, 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070413+0000", 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070413+0000", 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070412+0000", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5359999999999992e-06 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070420+0000", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4917e-05 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070435+0000", 2023-10-09T04:34:49.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7509999999999997e-06 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.070441+0000", 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0052465089999999999 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.075687+0000", 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9118000000000001e-05 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.075717+0000", 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0053034759999999997 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117881 6.4 e152/151)", 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.073325+0000", 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.013041677, 2023-10-09T04:34:49.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013284961, 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "queued for pg", 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073325+0000", 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073325+0000", 2023-10-09T04:34:49.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073323+0000", 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1406999999999999e-05 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073335+0000", 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7643e-05 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073352+0000", 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.126e-06 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.073359+0000", 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4619999999999997e-05 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.116 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117893 6.4 e152/151)", 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.082200+0000", 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0041663129999999996, 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044384810000000002, 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "queued for pg", 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082200+0000", 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.117 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082200+0000", 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082199+0000", 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1005999999999999e-05 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082210+0000", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9074999999999999e-05 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082229+0000", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3699999999999995e-06 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.082237+0000", 2023-10-09T04:34:49.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7197999999999999e-05 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:117887 6.c e152/151)", 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:49.084273+0000", 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0020934069999999998, 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023942, 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "queued for pg", 2023-10-09T04:34:49.120 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084273+0000", 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084273+0000", 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084272+0000", 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084283+0000", 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5865000000000001e-05 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:34:49.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084299+0000", 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.798e-06 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:49.084306+0000", 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2929000000000003e-05 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: ], 2023-10-09T04:34:49.123 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 8 2023-10-09T04:34:49.124 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:34:49.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:49.184 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:49.291 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:49.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:49.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:49.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:49.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:49.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:49.584 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:49 smithi153 bash[59565]: cluster 2023-10-09T04:34:48.349179+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v591: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 25744/176259 objects degraded (14.606%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:34:49.594 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:49 smithi082 bash[77063]: cluster 2023-10-09T04:34:48.349179+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v591: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 25744/176259 objects degraded (14.606%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:34:49.594 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:49 smithi082 bash[79733]: cluster 2023-10-09T04:34:48.349179+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v591: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 25744/176259 objects degraded (14.606%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:34:49.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:49.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:49.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:49.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:49.862 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:49] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T04:34:49.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:49.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:49.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:50.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:34:50.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:50.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:50.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:50.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:50 smithi082 bash[77063]: cluster 2023-10-09T04:34:49.578005+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 25744/176259 objects degraded (14.606%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:50.520 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:50.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:50.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.671 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:50 smithi153 bash[59565]: cluster 2023-10-09T04:34:49.578005+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 25744/176259 objects degraded (14.606%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:50.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:50 smithi082 bash[79733]: cluster 2023-10-09T04:34:49.578005+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 25744/176259 objects degraded (14.606%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:50.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:50.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:50.910 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:50.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.962 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:50.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:51.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:51.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:51.235 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:51.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:51.383 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:51.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:51.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:51.672 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:51.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.787 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:51 smithi153 bash[59565]: cluster 2023-10-09T04:34:50.349911+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v592: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:51.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:51.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:51.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:51.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:51.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:51.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:51 smithi082 bash[77063]: cluster 2023-10-09T04:34:50.349911+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v592: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:51.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:51 smithi082 bash[79733]: cluster 2023-10-09T04:34:50.349911+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v592: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:52.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:52.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:52.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:52.193 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:52.220 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:52.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:52.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:52.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:52.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:34:52.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:52.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:52.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:52.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:52.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:52.771 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:52.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:52.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:52.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:53.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:53.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:53.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:53.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:53.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:53.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:53.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:53.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:53.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:53.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:53 smithi082 bash[77063]: cluster 2023-10-09T04:34:52.350484+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v593: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:53.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:53 smithi082 bash[79733]: cluster 2023-10-09T04:34:52.350484+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v593: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:53.742 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:53.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:53.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:53.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:53 smithi153 bash[59565]: cluster 2023-10-09T04:34:52.350484+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v593: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25744/170595 objects degraded (15.091%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:34:53.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:54.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.148 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:54.250 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:54.273 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:54.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:54.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:54.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:54.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:54 smithi153 bash[59565]: audit 2023-10-09T04:34:53.750712+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:54.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:54 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.578927+0000 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 25744/170595 objects degraded (15.091%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:54.614 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:34:54.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.682 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:54.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.800 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:54 smithi082 bash[77063]: audit 2023-10-09T04:34:53.750712+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:54.800 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:54 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.578927+0000 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 25744/170595 objects degraded (15.091%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:54.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:54 smithi082 bash[79733]: audit 2023-10-09T04:34:53.750712+0000 mon.a (mon.0) 912 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:54.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:54 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.578927+0000 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 25744/170595 objects degraded (15.091%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:54.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:54.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:54.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:55.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:55.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:55.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:55.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:55.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:55.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:55.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:55.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:55.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:55.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.351101+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v594: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 25614/165351 objects degraded (15.491%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.587930+0000 mon.a (mon.0) 914 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.597528+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: audit 2023-10-09T04:34:54.600505+0000 mon.a (mon.0) 915 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.600689+0000 mon.a (mon.0) 916 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:34:55.755 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.609403+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.610104+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.611200+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[77063]: cluster 2023-10-09T04:34:54.611615+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.351101+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v594: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 25614/165351 objects degraded (15.491%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.587930+0000 mon.a (mon.0) 914 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.597528+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: audit 2023-10-09T04:34:54.600505+0000 mon.a (mon.0) 915 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.600689+0000 mon.a (mon.0) 916 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.609403+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.756 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.610104+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.757 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.611200+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.757 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:55 smithi082 bash[79733]: cluster 2023-10-09T04:34:54.611615+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.769 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:55.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:55.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:55.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:55.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.351101+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v594: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 25614/165351 objects degraded (15.491%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.587930+0000 mon.a (mon.0) 914 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.597528+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: audit 2023-10-09T04:34:54.600505+0000 mon.a (mon.0) 915 : audit 1 from='client.? 172.21.15.82:0/991532351' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.600689+0000 mon.a (mon.0) 916 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.609403+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.610104+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.611200+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:55.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:55 smithi153 bash[59565]: cluster 2023-10-09T04:34:54.611615+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T04:34:56.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:56.027 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:56.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:56.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:56.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:56.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:56.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:56.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:56.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:56.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:56.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:56.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:56.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:56.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:56.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:56.837 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:56.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:56.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:56.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:56.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:57.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:57.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:57.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:57.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:57.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:57.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:57.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:57.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:57.606 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:57 smithi082 bash[77063]: cluster 2023-10-09T04:34:56.351768+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v596: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25614/160485 objects degraded (15.960%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:34:57.606 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:57 smithi082 bash[79733]: cluster 2023-10-09T04:34:56.351768+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v596: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25614/160485 objects degraded (15.960%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:34:57.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:57.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:57.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:57.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:57.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:57.918 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:57.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:57.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:57 smithi153 bash[59565]: cluster 2023-10-09T04:34:56.351768+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v596: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25614/160485 objects degraded (15.960%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:34:58.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:58.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:58.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:58.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:58.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:58.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:58.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:58.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:58.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:58.666 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:58.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:58.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:58.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:58.981 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:59.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:59.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:59.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:59.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:59.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:59.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:59.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:59.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:59.430 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:59.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:59.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:59.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:59 smithi153 bash[59565]: cluster 2023-10-09T04:34:58.352417+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v597: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25484/155943 objects degraded (16.342%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:34:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:59 smithi153 bash[59565]: audit 2023-10-09T04:34:59.272493+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:59 smithi153 bash[59565]: audit 2023-10-09T04:34:59.273761+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:59.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:34:59 smithi153 bash[59565]: cluster 2023-10-09T04:34:59.580064+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 25484/155943 objects degraded (16.342%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:59.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:59.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:59.809 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[77063]: cluster 2023-10-09T04:34:58.352417+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v597: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25484/155943 objects degraded (16.342%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:34:59.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[77063]: audit 2023-10-09T04:34:59.272493+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:59.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[77063]: audit 2023-10-09T04:34:59.273761+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:59.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[77063]: cluster 2023-10-09T04:34:59.580064+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 25484/155943 objects degraded (16.342%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:59.810 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:34:59] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:34:59.810 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[79733]: cluster 2023-10-09T04:34:58.352417+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v597: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 25484/155943 objects degraded (16.342%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:34:59.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[79733]: audit 2023-10-09T04:34:59.272493+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:34:59.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[79733]: audit 2023-10-09T04:34:59.273761+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:59.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:34:59 smithi082 bash[79733]: cluster 2023-10-09T04:34:59.580064+0000 mon.a (mon.0) 919 : cluster 3 Health check update: Degraded data redundancy: 25484/155943 objects degraded (16.342%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:34:59.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:59.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:59.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:34:59.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:00.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:00.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:00.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:35:00.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:00.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.362 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:00.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:00.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:00.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:00.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:00.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:00.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:00.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:01.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:01.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:01.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:01.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:01.283 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:01.285 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:35:01.285 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:35:01.285 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:35:01.285 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34794.0:110524 5.0 5:0ed87686:::benchmark_data_smithi082_106363_object110523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 379.24259964800001, 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.098985219999999999, 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.286 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34794", 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 110524 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.287 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034303+0000", 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034302+0000", 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034327+0000", 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5131e-05 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034345+0000", 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7963999999999999e-05 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.034354+0000", 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2499999999999995e-06 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.038086+0000", 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0037325629999999999 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.039172+0000", 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001085637 2023-10-09T04:35:01.291 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.039582+0000", 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000409577 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.045532+0000", 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0059498820000000001 2023-10-09T04:35:01.292 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.051599+0000", 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060676209999999996 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133142+0000", 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.081543268000000002 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133244+0000", 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010136699999999999 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:28:42.133288+0000", 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4650000000000001e-05 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.294 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:8267 6.b e136/132 6:da1dc0e7:::benchmark_data_smithi082_128868_object8266:head v 136'482, mlcod=136'479)", 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:12.653454+0000", 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 168.62344909399999, 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.102292936, 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653454+0000", 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653454+0000", 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653453+0000", 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653481+0000", 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8042999999999999e-05 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653490+0000", 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0310000000000004e-06 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653497+0000", 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9040000000000003e-06 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653904+0000", 2023-10-09T04:35:01.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040676599999999997 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.653916+0000", 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2142e-05 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755716+0000", 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10179956 2023-10-09T04:35:01.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755747+0000", 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1015e-05 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:8272 6.b e136/132 6:d5fd91e8:::benchmark_data_smithi082_128868_object8271:head v 136'483, mlcod=136'479)", 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:12.654646+0000", 2023-10-09T04:35:01.300 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 168.62225661799999, 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.101514812, 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654646+0000", 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654646+0000", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654645+0000", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654672+0000", 2023-10-09T04:35:01.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7008e-05 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654683+0000", 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0288999999999999e-05 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654690+0000", 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3719999999999999e-06 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755191+0000", 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10050044800000001 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755227+0000", 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6541999999999997e-05 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756139+0000", 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00091140000000000004 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756161+0000", 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2531000000000002e-05 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:8273 6.7 e136/132 6:e47846f3:::benchmark_data_smithi082_128868_object8272:head v 136'533, mlcod=136'531)", 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:12.654885+0000", 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 168.62201784000001, 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.101562122, 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654885+0000", 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654885+0000", 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654884+0000", 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654927+0000", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2113999999999997e-05 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654935+0000", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0409999999999998e-06 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654939+0000", 2023-10-09T04:35:01.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9139999999999999e-06 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.654985+0000", 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5556999999999998e-05 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.655001+0000", 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5787999999999999e-05 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756378+0000", 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.101376709 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756447+0000", 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9627000000000003e-05 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.310 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:8269 6.f e136/132 6:f962efc1:::benchmark_data_smithi082_128868_object8268:head v 136'480, mlcod=136'477)", 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:32:12.655762+0000", 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 168.621141317, 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.100432283, 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.655762+0000", 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.655762+0000", 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.655761+0000", 2023-10-09T04:35:01.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.655786+0000", 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4998999999999999e-05 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755210+0000", 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.099424060999999994 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755233+0000", 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3346999999999999e-05 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755329+0000", 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5950000000000001e-05 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.755349+0000", 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0211e-05 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756166+0000", 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00081652899999999997 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:32:12.756194+0000", 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7917999999999999e-05 2023-10-09T04:35:01.315 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79268 6.0 e137/132 6:039ff843:::benchmark_data_smithi082_128868_object79267:head v 137'5087, mlcod=137'5083)", 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.583135+0000", 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.69376749, 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12019193, 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583135+0000", 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583135+0000", 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583135+0000", 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583155+0000", 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0809e-05 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583167+0000", 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1402e-05 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583265+0000", 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8282999999999994e-05 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583306+0000", 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0961999999999998e-05 2023-10-09T04:35:01.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583319+0000", 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2724999999999999e-05 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703293+0000", 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11997469600000001 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703327+0000", 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3883000000000001e-05 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79269 6.0 e137/132 6:0737ecb6:::benchmark_data_smithi082_128868_object79268:head v 137'5088, mlcod=137'5083)", 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.583530+0000", 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.69337315200001, 2023-10-09T04:35:01.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.119897845, 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583530+0000", 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583530+0000", 2023-10-09T04:35:01.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583529+0000", 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583556+0000", 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7325e-05 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583568+0000", 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1063999999999999e-05 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.583575+0000", 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.486e-06 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.587546+0000", 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039714099999999999 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.587561+0000", 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4229999999999999e-05 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703359+0000", 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.115798123 2023-10-09T04:35:01.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703428+0000", 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8790999999999999e-05 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.326 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79274 6.4 e137/132 6:2801f20d:::benchmark_data_smithi082_128868_object79273:head v 137'4948, mlcod=137'4946)", 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.585101+0000", 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.69180201499999, 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.108787731, 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585101+0000", 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585101+0000", 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585100+0000", 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585129+0000", 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8643e-05 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585140+0000", 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1525000000000001e-05 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585147+0000", 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1790000000000002e-06 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.588731+0000", 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0035836729999999999 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.588754+0000", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2699000000000001e-05 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693739+0000", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10498568899999999 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693889+0000", 2023-10-09T04:35:01.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014923200000000001 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79273 6.8 e137/132 6:1e87fb54:::benchmark_data_smithi082_128868_object79272:head v 137'5024, mlcod=137'5021)", 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.585190+0000", 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.691712825, 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.118166469, 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.332 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585190+0000", 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585190+0000", 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585190+0000", 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585217+0000", 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7044999999999999e-05 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585223+0000", 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2060000000000004e-06 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585228+0000", 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2270000000000001e-06 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585968+0000", 2023-10-09T04:35:01.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00074010999999999999 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.585989+0000", 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0815e-05 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703335+0000", 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11734597300000001 2023-10-09T04:35:01.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.703357+0000", 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1452999999999999e-05 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79278 6.6 e137/132 6:6f63f2fc:::benchmark_data_smithi082_128868_object79277:head v 137'4831, mlcod=137'4829)", 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.589088+0000", 2023-10-09T04:35:01.337 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.68781474799999, 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.104678035, 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589088+0000", 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589088+0000", 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589054+0000", 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999657 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589132+0000", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8472000000000004e-05 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589153+0000", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1084999999999999e-05 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589162+0000", 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4339999999999992e-06 2023-10-09T04:35:01.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589207+0000", 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5422e-05 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589226+0000", 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9108999999999999e-05 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693664+0000", 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.104437617 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693766+0000", 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000102104 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.342 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79276 6.7 e137/132 6:e35e5943:::benchmark_data_smithi082_128868_object79275:head v 137'4967, mlcod=137'4965)", 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.589175+0000", 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.6877278, 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.104598713, 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589175+0000", 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589175+0000", 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589175+0000", 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589206+0000", 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1359000000000002e-05 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589213+0000", 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.7299999999999999e-06 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589218+0000", 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2989999999999996e-06 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589284+0000", 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5566000000000002e-05 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.589305+0000", 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1454000000000002e-05 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693668+0000", 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.104363311 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693774+0000", 2023-10-09T04:35:01.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000105345 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79284 6.5 e137/132 6:a2c23b4d:::benchmark_data_smithi082_128868_object79283:head v 137'4904, mlcod=137'4902)", 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.590809+0000", 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.686093593, 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10746478399999999, 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.348 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590809+0000", 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590809+0000", 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590809+0000", 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590836+0000", 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6877e-05 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590846+0000", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0798000000000001e-05 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590855+0000", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6100000000000006e-06 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693624+0000", 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.102768861 2023-10-09T04:35:01.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693684+0000", 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0349000000000001e-05 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.698193+0000", 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0045087469999999996 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.698274+0000", 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1073000000000001e-05 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79281 6.0 e137/132 6:02788de3:::benchmark_data_smithi082_128868_object79280:head v 137'5089, mlcod=137'5084)", 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.590883+0000", 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.686020164, 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11841278199999999, 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.353 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590883+0000", 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590883+0000", 2023-10-09T04:35:01.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590882+0000", 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590902+0000", 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0040000000000001e-05 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590911+0000", 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5110000000000003e-06 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.590916+0000", 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7219999999999999e-06 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.694079+0000", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10316312900000001 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.694100+0000", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1460000000000001e-05 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.709259+0000", 2023-10-09T04:35:01.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.015158712 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.709295+0000", 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6587000000000001e-05 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.360 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79283 6.7 e137/132 6:efa62da7:::benchmark_data_smithi082_128868_object79282:head v 137'4968, mlcod=137'4965)", 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.592131+0000", 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.684771536, 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.107589685, 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.592131+0000", 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.592131+0000", 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.592131+0000", 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.592225+0000", 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4611999999999995e-05 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693755+0000", 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.101529841 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693772+0000", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7513e-05 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.694056+0000", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00028315099999999999 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.694103+0000", 2023-10-09T04:35:01.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7777e-05 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.699559+0000", 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0054559400000000003 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.699721+0000", 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016168500000000001 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.365 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79282 6.f e137/132 6:f4f16db7:::benchmark_data_smithi082_128868_object79281:head v 137'4966, mlcod=137'4964)", 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:01.593031+0000", 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.683872231, 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.106520789, 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.593031+0000", 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.593031+0000", 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.593030+0000", 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.593050+0000", 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9905e-05 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693590+0000", 2023-10-09T04:35:01.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10053973500000001 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693604+0000", 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4095e-05 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693643+0000", 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9410999999999999e-05 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.693683+0000", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9502999999999998e-05 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.699357+0000", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00567418 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:01.699551+0000", 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000194716 2023-10-09T04:35:01.370 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:79637 6.c e138/132 6:36b0c9e6:::benchmark_data_smithi082_128868_object79636:head v 138'5011, mlcod=138'5008)", 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:02.007001+0000", 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 119.269901389, 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.100092061, 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.371 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007001+0000", 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007001+0000", 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007001+0000", 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007027+0000", 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5412999999999999e-05 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007032+0000", 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.007035+0000", 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.022e-06 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.016954+0000", 2023-10-09T04:35:01.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0099188550000000007 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.016976+0000", 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1936000000000001e-05 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.107066+0000", 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.090090365000000006 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:02.107094+0000", 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.76e-05 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87206 6.c 6.51e6993c (undecoded) ondisk+retry+write+known_if_redirected e139)", 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:35:01.376 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 111.534097984, 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.019224345, 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87206 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:35:01.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742804+0000", 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742829+0000", 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5610999999999999e-05 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742837+0000", 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0819999999999999e-06 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742845+0000", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2069999999999998e-06 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742889+0000", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4777000000000002e-05 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op must wait for map", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742894+0000", 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8300000000000003e-06 2023-10-09T04:35:01.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.755779+0000", 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012884678 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.755788+0000", 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8890000000000008e-06 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.761987+0000", 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0061995909999999 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762029+0000", 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1962999999999998e-05 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.382 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87233 6.c 6.8e484a9c (undecoded) ondisk+write+known_if_redirected e139)", 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 111.50363882400001, 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98904841899999996, 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87233 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.383 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773262+0000", 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773303+0000", 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0358000000000002e-05 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773320+0000", 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6753999999999999e-05 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773328+0000", 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773415+0000", 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7342000000000005e-05 2023-10-09T04:35:01.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773432+0000", 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7217000000000001e-05 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762301+0000", 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98886854599999996 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.387 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762312+0000", 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1591e-05 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.388 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87251 6.c 6.ab78b1bc (undecoded) ondisk+write+known_if_redirected e139)", 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 111.476098849, 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96155623000000001, 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87251 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800802+0000", 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800875+0000", 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3133999999999995e-05 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800911+0000", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6018e-05 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800964+0000", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2862999999999999e-05 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.801027+0000", 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3337999999999996e-05 2023-10-09T04:35:01.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.801042+0000", 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5002e-05 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762351+0000", 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96130837400000002 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762360+0000", 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3950000000000007e-06 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.34995.0:92703 6.c e147/146 6:3977e8ae:::benchmark_data_smithi082_128868_object5365:head v 147'5868, mlcod=147'5865)", 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:34:16.258661+0000", 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 45.018241644, 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.098291356999999996, 2023-10-09T04:35:01.394 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.258661+0000", 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.258661+0000", 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.258660+0000", 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:35:01.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.258668+0000", 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0700000000000007e-06 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.332474+0000", 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.073805307000000001 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.332484+0000", 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0487e-05 2023-10-09T04:35:01.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.333261+0000", 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00077678199999999995 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:35:01.398 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.333285+0000", 2023-10-09T04:35:01.402 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4128999999999999e-05 2023-10-09T04:35:01.402 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.402 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.402 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.356926+0000", 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023641154000000001 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:34:16.356953+0000", 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6378000000000001e-05 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:35:01.403 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:35:01.404 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:35:01.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:01.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:01.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:01.529 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:01.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:01.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:01.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:01.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:01.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.807 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:01 smithi082 bash[77063]: cluster 2023-10-09T04:35:00.353130+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v598: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:01.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:01.807 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:01 smithi082 bash[79733]: cluster 2023-10-09T04:35:00.353130+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v598: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:01.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:01.917 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:01 smithi153 bash[59565]: cluster 2023-10-09T04:35:00.353130+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v598: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:01.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:01.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:02.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:02.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:02.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:02.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:02.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:02.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:02.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:02.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:02.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:02.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:02.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:02.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:02.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:02.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:02.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:03.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:03.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:03.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:03.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:03.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:03.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:03.416 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:03.450 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:03.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:03.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:03.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:03 smithi153 bash[59565]: cluster 2023-10-09T04:35:02.353660+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v599: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:03.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:03.698 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:03 smithi082 bash[79733]: cluster 2023-10-09T04:35:02.353660+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v599: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:03.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:03 smithi082 bash[77063]: cluster 2023-10-09T04:35:02.353660+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v599: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 25484/150147 objects degraded (16.973%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:35:03.727 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:03.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:03.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:03.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:03.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:03.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:03.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:04.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:04.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:04.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:04.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:04.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:04.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:04.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:04.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:04.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:04.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:04.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:04.637 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:04 smithi082 bash[77063]: cluster 2023-10-09T04:35:04.581262+0000 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 25484/150147 objects degraded (16.973%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:04.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:04.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:04.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:04.812 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:04 smithi153 bash[59565]: cluster 2023-10-09T04:35:04.581262+0000 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 25484/150147 objects degraded (16.973%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:04.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:04.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:35:04.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:04.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:04.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:04.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:04 smithi082 bash[79733]: cluster 2023-10-09T04:35:04.581262+0000 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 25484/150147 objects degraded (16.973%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:05.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:05.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:05.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:05.266 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:05.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:05.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:05.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:05.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:05.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:05.650 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:05 smithi153 bash[59565]: cluster 2023-10-09T04:35:04.354383+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v600: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 25340/144927 objects degraded (17.485%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:05.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:05.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:05.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:05.894 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:05.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:05.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:05.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:05 smithi082 bash[77063]: cluster 2023-10-09T04:35:04.354383+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v600: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 25340/144927 objects degraded (17.485%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:05 smithi082 bash[79733]: cluster 2023-10-09T04:35:04.354383+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v600: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 25340/144927 objects degraded (17.485%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:06.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:06.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:06.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:06.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:06.294 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:06.326 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:06.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:06.469 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:06.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:06.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:06.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:06.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:06.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:06.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:06.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:06.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:07.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:07.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:07.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:07.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:07.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.447 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:07.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:07.577 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:07.653 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:07 smithi082 bash[77063]: cluster 2023-10-09T04:35:06.355094+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v601: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 25340/140199 objects degraded (18.074%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:35:07.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:07 smithi153 bash[59565]: cluster 2023-10-09T04:35:06.355094+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v601: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 25340/140199 objects degraded (18.074%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:35:07.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:07.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:07.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:07.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:07.907 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:07 smithi082 bash[79733]: cluster 2023-10-09T04:35:06.355094+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v601: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 25340/140199 objects degraded (18.074%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:35:07.969 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:08.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:08.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:08.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:08.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:08.187 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:35:08.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.252 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:35:08.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:08.299 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:35:08.338 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:35:08.378 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:35:08.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.389 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:08.414 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:35:08.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:08.456 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:35:08.488 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.493 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:08.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:08.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:08.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:08.668 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:08 smithi082 bash[77063]: audit 2023-10-09T04:35:08.126377+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/1363962777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:08.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:08.698 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:08.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:08 smithi082 bash[79733]: audit 2023-10-09T04:35:08.126377+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/1363962777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:08.806 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:08.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:08.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:08 smithi153 bash[59565]: audit 2023-10-09T04:35:08.126377+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.82:0/1363962777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:08.969 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:09.001 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:09.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:09.140 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:09.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:09.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:09.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:09.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:09.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:09.532 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:09.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:09.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.672 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[77063]: cluster 2023-10-09T04:35:08.355719+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v602: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 25214/135762 objects degraded (18.572%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:35:09.672 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[77063]: audit 2023-10-09T04:35:08.769728+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:35:09.672 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[77063]: cluster 2023-10-09T04:35:09.582983+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 25214/135762 objects degraded (18.572%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:09.673 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[79733]: cluster 2023-10-09T04:35:08.355719+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v602: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 25214/135762 objects degraded (18.572%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:35:09.673 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[79733]: audit 2023-10-09T04:35:08.769728+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:35:09.673 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[79733]: cluster 2023-10-09T04:35:09.582983+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 25214/135762 objects degraded (18.572%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:09.678 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:09.685 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:35:09.714 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:09.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:09.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:09.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:09 smithi153 bash[59565]: cluster 2023-10-09T04:35:08.355719+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v602: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 25214/135762 objects degraded (18.572%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:35:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:09 smithi153 bash[59565]: audit 2023-10-09T04:35:08.769728+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:35:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:09 smithi153 bash[59565]: cluster 2023-10-09T04:35:09.582983+0000 mon.a (mon.0) 922 : cluster 3 Health check update: Degraded data redundancy: 25214/135762 objects degraded (18.572%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:09.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:09.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:09] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:35:10.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:10.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.238 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:10.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:10.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:10.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:10.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:10.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:10.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:10.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:10.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[79733]: audit 2023-10-09T04:35:09.679908+0000 mon.a (mon.0) 923 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[79733]: cluster 2023-10-09T04:35:09.679988+0000 mon.a (mon.0) 924 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[79733]: cluster 2023-10-09T04:35:09.685609+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[79733]: cluster 2023-10-09T04:35:09.691347+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[79733]: cluster 2023-10-09T04:35:09.693071+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.932 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[77063]: audit 2023-10-09T04:35:09.679908+0000 mon.a (mon.0) 923 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:35:10.933 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[77063]: cluster 2023-10-09T04:35:09.679988+0000 mon.a (mon.0) 924 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:35:10.933 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[77063]: cluster 2023-10-09T04:35:09.685609+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.933 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[77063]: cluster 2023-10-09T04:35:09.691347+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.933 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:10 smithi082 bash[77063]: cluster 2023-10-09T04:35:09.693071+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:10.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:10 smithi153 bash[59565]: audit 2023-10-09T04:35:09.679908+0000 mon.a (mon.0) 923 : audit 1 from='client.? 172.21.15.82:0/1545323925' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:35:10.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:10 smithi153 bash[59565]: cluster 2023-10-09T04:35:09.679988+0000 mon.a (mon.0) 924 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:35:10.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:10 smithi153 bash[59565]: cluster 2023-10-09T04:35:09.685609+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.946 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:10 smithi153 bash[59565]: cluster 2023-10-09T04:35:09.691347+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.947 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:10 smithi153 bash[59565]: cluster 2023-10-09T04:35:09.693071+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T04:35:10.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:11.048 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:11.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:11.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:11.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:11.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:11.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:11.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:11.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.614 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:11.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:11.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.878 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:11.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:11 smithi082 bash[77063]: cluster 2023-10-09T04:35:10.356417+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v604: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:11.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:11 smithi082 bash[79733]: cluster 2023-10-09T04:35:10.356417+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v604: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:11.985 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:12.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:12.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:12.143 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:11 smithi153 bash[59565]: cluster 2023-10-09T04:35:10.356417+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v604: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:12.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:12.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:12.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:12.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:12.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:12.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:12.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:12.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:35:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:12.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:12.766 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:12.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:12.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:12.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:12.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:13.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:13.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:13.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:13.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:13.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:13.600 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:13.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:13.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:13.840 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:13.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:13.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:14.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:14.032 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:13 smithi153 bash[59565]: cluster 2023-10-09T04:35:12.356889+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v605: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:14.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:14.122 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:13 smithi082 bash[77063]: cluster 2023-10-09T04:35:12.356889+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v605: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:14.122 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:13 smithi082 bash[79733]: cluster 2023-10-09T04:35:12.356889+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v605: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25214/130338 objects degraded (19.345%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:35:14.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:14.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:14.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:14.343 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:14.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:14.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:14.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:14.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:14.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:14.669 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:14.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:14.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:14.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:14.813 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:14 smithi082 bash[77063]: audit 2023-10-09T04:35:14.267668+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:14.813 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:14 smithi082 bash[77063]: cluster 2023-10-09T04:35:14.585350+0000 mon.a (mon.0) 926 : cluster 3 Health check update: Degraded data redundancy: 25214/130338 objects degraded (19.345%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:14.813 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:14 smithi082 bash[79733]: audit 2023-10-09T04:35:14.267668+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:14.813 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:14 smithi082 bash[79733]: cluster 2023-10-09T04:35:14.585350+0000 mon.a (mon.0) 926 : cluster 3 Health check update: Degraded data redundancy: 25214/130338 objects degraded (19.345%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:14.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:14.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:14.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:14.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:15.049 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:14 smithi153 bash[59565]: audit 2023-10-09T04:35:14.267668+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:15.050 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:14 smithi153 bash[59565]: cluster 2023-10-09T04:35:14.585350+0000 mon.a (mon.0) 926 : cluster 3 Health check update: Degraded data redundancy: 25214/130338 objects degraded (19.345%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:15.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:15.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:15.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:15.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:15.333 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:15.333 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:15.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:15.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:15.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:15.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:15.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:15.642 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:15.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:15.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:15.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:15.892 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:15 smithi153 bash[59565]: cluster 2023-10-09T04:35:14.357603+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v606: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 25079/125559 objects degraded (19.974%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:15.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:15.921 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:15 smithi082 bash[77063]: cluster 2023-10-09T04:35:14.357603+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v606: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 25079/125559 objects degraded (19.974%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:15.921 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:15 smithi082 bash[79733]: cluster 2023-10-09T04:35:14.357603+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v606: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 25079/125559 objects degraded (19.974%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:15.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:16.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:16.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:16.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:16.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:35:16.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:16.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.614 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:16.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:16.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.745 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:35:16.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:16.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:16.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:16.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:16 smithi082 bash[77063]: audit 2023-10-09T04:35:16.583490+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:16.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:16 smithi082 bash[79733]: audit 2023-10-09T04:35:16.583490+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:16.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:17.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:17.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:17.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:17.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:16 smithi153 bash[59565]: audit 2023-10-09T04:35:16.583490+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:17.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:17.287 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:17.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:17.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:17.421 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:17.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:17.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:17.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:17.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:17.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:17.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:17.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: cluster 2023-10-09T04:35:16.358715+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v607: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25079/120612 objects degraded (20.793%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:17.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: cluster 2023-10-09T04:35:16.710054+0000 mon.a (mon.0) 928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: cluster 2023-10-09T04:35:16.725490+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: cluster 2023-10-09T04:35:16.728088+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: audit 2023-10-09T04:35:16.735873+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[77063]: cluster 2023-10-09T04:35:16.736493+0000 mon.a (mon.0) 930 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: cluster 2023-10-09T04:35:16.358715+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v607: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25079/120612 objects degraded (20.793%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: cluster 2023-10-09T04:35:16.710054+0000 mon.a (mon.0) 928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: cluster 2023-10-09T04:35:16.725490+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: cluster 2023-10-09T04:35:16.728088+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: audit 2023-10-09T04:35:16.735873+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:17.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:17 smithi082 bash[79733]: cluster 2023-10-09T04:35:16.736493+0000 mon.a (mon.0) 930 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:35:17.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:17.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:17.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:17.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:18.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: cluster 2023-10-09T04:35:16.358715+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v607: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25079/120612 objects degraded (20.793%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: cluster 2023-10-09T04:35:16.710054+0000 mon.a (mon.0) 928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: cluster 2023-10-09T04:35:16.725490+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: cluster 2023-10-09T04:35:16.728088+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: audit 2023-10-09T04:35:16.735873+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.82:0/3496992249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:18.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:17 smithi153 bash[59565]: cluster 2023-10-09T04:35:16.736493+0000 mon.a (mon.0) 930 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:35:18.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:18.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:18.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:18.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:18.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:18.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:18.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:18.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:18.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:18.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:18.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:18.746 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:18.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:18.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:18.914 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:18.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:18.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:19.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:19.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:19.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:19.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:19.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:19.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:19.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:19.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:19.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:19.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:19.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:19.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:19.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:19.933 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:19.965 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:19.973 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:19 smithi082 bash[77063]: cluster 2023-10-09T04:35:18.359388+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v609: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24939/116343 objects degraded (21.436%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:35:19.973 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:19 smithi082 bash[77063]: cluster 2023-10-09T04:35:19.586517+0000 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 24939/116343 objects degraded (21.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:19.973 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:19] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:35:19.974 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:19 smithi082 bash[79733]: cluster 2023-10-09T04:35:18.359388+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v609: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24939/116343 objects degraded (21.436%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:35:19.974 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:19 smithi082 bash[79733]: cluster 2023-10-09T04:35:19.586517+0000 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 24939/116343 objects degraded (21.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:20.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.007 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:19 smithi153 bash[59565]: cluster 2023-10-09T04:35:18.359388+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v609: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24939/116343 objects degraded (21.436%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:35:20.007 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:19 smithi153 bash[59565]: cluster 2023-10-09T04:35:19.586517+0000 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 24939/116343 objects degraded (21.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:20.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:20.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:20.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:20.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:20.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:20.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:20.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:20.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:35:20.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:20.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:20.665 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:20.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:20.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:20.850 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:20.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:20.954 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:20.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:21.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:21.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:21.205 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:21.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:21.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:21.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:21.518 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:21.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:21.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:21.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:21.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.942 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:21 smithi082 bash[79733]: cluster 2023-10-09T04:35:20.360188+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v610: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:21.942 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:21 smithi082 bash[77063]: cluster 2023-10-09T04:35:20.360188+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v610: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:21.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:21.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:21.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:22.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:22.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:21 smithi153 bash[59565]: cluster 2023-10-09T04:35:20.360188+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v610: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:22.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:22.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:22.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:22.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:22.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:22.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:22.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:22.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:22.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:22.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:22.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:22 smithi082 bash[77063]: audit 2023-10-09T04:35:22.428178+0000 mon.a (mon.0) 932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:22.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:22.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:22.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:22.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:23.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:23.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:23.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:22 smithi082 bash[79733]: audit 2023-10-09T04:35:22.428178+0000 mon.a (mon.0) 932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:23.152 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:22 smithi153 bash[59565]: audit 2023-10-09T04:35:22.428178+0000 mon.a (mon.0) 932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:23.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:23.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:23.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:23.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:23.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:23.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:23.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:23.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:23.866 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:23 smithi082 bash[77063]: cluster 2023-10-09T04:35:22.360654+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v611: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:23.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:23 smithi082 bash[79733]: cluster 2023-10-09T04:35:22.360654+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v611: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:23.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:23.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:23.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:23 smithi153 bash[59565]: cluster 2023-10-09T04:35:22.360654+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v611: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 24939/111147 objects degraded (22.438%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:24.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:24.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:24.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:24.172 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:24.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:24.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:24.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:24.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:24.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:24.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:24.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:24.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:24.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:24.821 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:24.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:24.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:24 smithi082 bash[77063]: cluster 2023-10-09T04:35:24.587842+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 24939/111147 objects degraded (22.438%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:24.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:24 smithi082 bash[79733]: cluster 2023-10-09T04:35:24.587842+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 24939/111147 objects degraded (22.438%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:24.989 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:24.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:25.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:24 smithi153 bash[59565]: cluster 2023-10-09T04:35:24.587842+0000 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 24939/111147 objects degraded (22.438%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:25.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:25.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:25.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:25.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:25.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.553 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:25.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:25.754 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:25.754 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:25.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:25.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:25.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:25 smithi082 bash[77063]: cluster 2023-10-09T04:35:24.361419+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v612: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24807/106254 objects degraded (23.347%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:25.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:25 smithi082 bash[79733]: cluster 2023-10-09T04:35:24.361419+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v612: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24807/106254 objects degraded (23.347%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:25.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:26.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:26.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:26.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:26.124 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:25 smithi153 bash[59565]: cluster 2023-10-09T04:35:24.361419+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v612: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24807/106254 objects degraded (23.347%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:26.214 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:26.319 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:26.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:26.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:26.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:26.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:26.533 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:26.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:26.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:26.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:26.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:35:26.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:26.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:26.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:26.942 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:26.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:27.051 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:27.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:27.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:27.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:27.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:27.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:27.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:27.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:27.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:27.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:27.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:27.767 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:27 smithi082 bash[77063]: cluster 2023-10-09T04:35:26.365667+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v613: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24807/101649 objects degraded (24.405%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:27.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:27.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:27.892 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:27 smithi082 bash[79733]: cluster 2023-10-09T04:35:26.365667+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v613: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24807/101649 objects degraded (24.405%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:27.913 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:27.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:27.935 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:35:27.974 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:35:28.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:28.021 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:35:28.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:27 smithi153 bash[59565]: cluster 2023-10-09T04:35:26.365667+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v613: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24807/101649 objects degraded (24.405%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:28.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:28.060 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:35:28.095 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:35:28.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:28.160 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:35:28.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:28.199 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:35:28.236 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:35:28.242 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.243 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:35:28.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:28.362 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:28.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:28.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:28.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:28.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:28.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:28.741 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:28.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:28.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:28.923 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:28.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:28.946 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:28 smithi082 bash[77063]: audit 2023-10-09T04:35:27.878283+0000 mon.a (mon.0) 934 : audit 0 from='client.? 172.21.15.82:0/3912792358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:28.946 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:28 smithi082 bash[79733]: audit 2023-10-09T04:35:27.878283+0000 mon.a (mon.0) 934 : audit 0 from='client.? 172.21.15.82:0/3912792358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:29.032 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:29.044 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:28 smithi153 bash[59565]: audit 2023-10-09T04:35:27.878283+0000 mon.a (mon.0) 934 : audit 0 from='client.? 172.21.15.82:0/3912792358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:29.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:29.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:29.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:29.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:29.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:29.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:29.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:29.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:29.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:29.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:29] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:35:29.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:29.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:29.855 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:35:29.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:29.957 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: cluster 2023-10-09T04:35:28.366613+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v614: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 24662/97737 objects degraded (25.233%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: audit 2023-10-09T04:35:28.791341+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: audit 2023-10-09T04:35:28.849180+0000 mon.a (mon.0) 936 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: audit 2023-10-09T04:35:28.857798+0000 mon.a (mon.0) 937 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: audit 2023-10-09T04:35:29.275833+0000 mon.a (mon.0) 938 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: audit 2023-10-09T04:35:29.277610+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[77063]: cluster 2023-10-09T04:35:29.589641+0000 mon.a (mon.0) 940 : cluster 3 Health check update: Degraded data redundancy: 24662/97737 objects degraded (25.233%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: cluster 2023-10-09T04:35:28.366613+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v614: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 24662/97737 objects degraded (25.233%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: audit 2023-10-09T04:35:28.791341+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:35:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: audit 2023-10-09T04:35:28.849180+0000 mon.a (mon.0) 936 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: audit 2023-10-09T04:35:28.857798+0000 mon.a (mon.0) 937 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: audit 2023-10-09T04:35:29.275833+0000 mon.a (mon.0) 938 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: audit 2023-10-09T04:35:29.277610+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:29.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:29 smithi082 bash[79733]: cluster 2023-10-09T04:35:29.589641+0000 mon.a (mon.0) 940 : cluster 3 Health check update: Degraded data redundancy: 24662/97737 objects degraded (25.233%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:29.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:29.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:30.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: cluster 2023-10-09T04:35:28.366613+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v614: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 24662/97737 objects degraded (25.233%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: audit 2023-10-09T04:35:28.791341+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: audit 2023-10-09T04:35:28.849180+0000 mon.a (mon.0) 936 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: audit 2023-10-09T04:35:28.857798+0000 mon.a (mon.0) 937 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: audit 2023-10-09T04:35:29.275833+0000 mon.a (mon.0) 938 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: audit 2023-10-09T04:35:29.277610+0000 mon.a (mon.0) 939 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:30.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:29 smithi153 bash[59565]: cluster 2023-10-09T04:35:29.589641+0000 mon.a (mon.0) 940 : cluster 3 Health check update: Degraded data redundancy: 24662/97737 objects degraded (25.233%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:30.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:30.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:30.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:30.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:30.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:30.680 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:30.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:30.785 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:30.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:29.798085+0000 mon.a (mon.0) 941 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: cluster 2023-10-09T04:35:29.798209+0000 mon.a (mon.0) 942 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: cluster 2023-10-09T04:35:29.799734+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: cluster 2023-10-09T04:35:29.816330+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: cluster 2023-10-09T04:35:29.818465+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:30.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:29.863731+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:29.875759+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:30.454708+0000 mon.a (mon.0) 945 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:30.456109+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:30.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:30 smithi153 bash[59565]: audit 2023-10-09T04:35:30.462970+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:30.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:30.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:30.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:31.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:31.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:31.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:29.798085+0000 mon.a (mon.0) 941 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: cluster 2023-10-09T04:35:29.798209+0000 mon.a (mon.0) 942 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: cluster 2023-10-09T04:35:29.799734+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: cluster 2023-10-09T04:35:29.816330+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: cluster 2023-10-09T04:35:29.818465+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:29.863731+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:29.875759+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:30.454708+0000 mon.a (mon.0) 945 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:30.456109+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[77063]: audit 2023-10-09T04:35:30.462970+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:29.798085+0000 mon.a (mon.0) 941 : audit 1 from='client.? 172.21.15.82:0/2855996579' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: cluster 2023-10-09T04:35:29.798209+0000 mon.a (mon.0) 942 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: cluster 2023-10-09T04:35:29.799734+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: cluster 2023-10-09T04:35:29.816330+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: cluster 2023-10-09T04:35:29.818465+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T04:35:31.199 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:29.863731+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.199 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:29.875759+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.199 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:30.454708+0000 mon.a (mon.0) 945 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:35:31.199 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:30.456109+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:35:31.199 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:30 smithi082 bash[79733]: audit 2023-10-09T04:35:30.462970+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:35:31.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:31.268 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:31.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:31.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:31.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:31.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:31.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:31.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:31.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:31.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:31.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:31.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:31.866 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:31.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:31.978 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:31 smithi153 bash[59565]: cluster 2023-10-09T04:35:30.371689+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v616: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:31.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:31.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:31.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:32.100 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:32.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:32.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:35:32.223 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:31 smithi082 bash[77063]: cluster 2023-10-09T04:35:30.371689+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v616: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:32.223 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:31 smithi082 bash[79733]: cluster 2023-10-09T04:35:30.371689+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v616: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:32.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:32.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:32.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:32.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:32.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:32.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:32.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:35:32.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:32.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:32.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:32.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:32.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:32.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:32.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:33.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:33.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:33.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:33.238 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:33.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:33.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:33.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:33.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:33.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:33.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:33.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:33.792 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:33.883 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:33 smithi153 bash[59565]: cluster 2023-10-09T04:35:32.377454+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v617: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:33.897 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:33.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:33.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:34.036 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:34.072 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:34.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:33 smithi082 bash[77063]: cluster 2023-10-09T04:35:32.377454+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v617: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:34.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:33 smithi082 bash[79733]: cluster 2023-10-09T04:35:32.377454+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v617: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 24662/92898 objects degraded (26.547%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:34.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:34.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:34.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:34.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:34.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:34.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:34.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:34.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:34.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:34.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:34.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:34.884 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:34 smithi082 bash[77063]: cluster 2023-10-09T04:35:34.590635+0000 mon.a (mon.0) 948 : cluster 3 Health check update: Degraded data redundancy: 24662/92898 objects degraded (26.547%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:34.884 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:34 smithi082 bash[77063]: audit 2023-10-09T04:35:34.843053+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:34.884 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:34 smithi082 bash[79733]: cluster 2023-10-09T04:35:34.590635+0000 mon.a (mon.0) 948 : cluster 3 Health check update: Degraded data redundancy: 24662/92898 objects degraded (26.547%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:34.885 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:34 smithi082 bash[79733]: audit 2023-10-09T04:35:34.843053+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:34.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:34.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:34.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:35.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:35.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:34 smithi153 bash[59565]: cluster 2023-10-09T04:35:34.590635+0000 mon.a (mon.0) 948 : cluster 3 Health check update: Degraded data redundancy: 24662/92898 objects degraded (26.547%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:35.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:34 smithi153 bash[59565]: audit 2023-10-09T04:35:34.843053+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:35.095 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:35.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:35.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:35.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:35.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:35.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:35.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:35.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:35.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:35.620 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:35:35.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:35.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:35.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:35.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:35.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20168 5.e 5:73f6c3c2:::benchmark_data_smithi082_106363_object20167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:35:35.914 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 492.047365673, 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12474445000000001, 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20168 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859969+0000", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859968+0000", 2023-10-09T04:35:35.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8635e-05 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859986+0000", 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4649999999999999e-06 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859993+0000", 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7000000000000002e-06 2023-10-09T04:35:35.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.859996+0000", 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3295999999999999e-05 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860050+0000", 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014821499999999999 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860198+0000", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0001594 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.860357+0000", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024722350000000001 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.862830+0000", 2023-10-09T04:35:35.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0014452270000000001 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.864275+0000", 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.120165037 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984440+0000", 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017054900000000001 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984610+0000", 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000102767 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984713+0000", 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12474445000000001 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:20184 5.e 5:7994a2da:::benchmark_data_smithi082_106363_object20183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 492.03808461099999, 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115792874, 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.922 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 20184 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869250+0000", 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999948 2023-10-09T04:35:35.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869244+0000", 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2075000000000004e-05 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869317+0000", 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2318e-05 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869329+0000", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5759999999999996e-06 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869334+0000", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2394000000000001e-05 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869377+0000", 2023-10-09T04:35:35.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010204600000000001 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869479+0000", 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011170100000000001 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.869591+0000", 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000800163 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.870391+0000", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00032963699999999997 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.870720+0000", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114094243 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984815+0000", 2023-10-09T04:35:35.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000125501 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.984940+0000", 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000102484 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:23.985043+0000", 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115792874 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.928 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71654 5.e 5:7baa3dd3:::benchmark_data_smithi082_106363_object71653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 454.70170429799998, 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115496826, 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.929 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71654 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205630+0000", 2023-10-09T04:35:35.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205629+0000", 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5355000000000002e-05 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205675+0000", 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3310000000000004e-06 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205683+0000", 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3079999999999998e-06 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205688+0000", 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029010419999999999 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.208589+0000", 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015918799999999999 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.208748+0000", 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027670860000000002 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.211515+0000", 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0043369649999999999 2023-10-09T04:35:35.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.215852+0000", 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00051018199999999997 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216363+0000", 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.104664519 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321027+0000", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1848999999999998e-05 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321079+0000", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7880999999999998e-05 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321127+0000", 2023-10-09T04:35:35.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115496826 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:71657 5.e 5:7b75b376:::benchmark_data_smithi082_106363_object71656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 454.70163952799999, 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115567221, 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.936 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 71657 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205695+0000", 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205694+0000", 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5553000000000003e-05 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205730+0000", 2023-10-09T04:35:35.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2259999999999999e-06 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205737+0000", 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2429999999999996e-06 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.205742+0000", 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080320359999999993 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213774+0000", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103989 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213878+0000", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010159500000000001 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.213980+0000", 2023-10-09T04:35:35.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002066713 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.216047+0000", 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0037985639999999999 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.219845+0000", 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.101326286 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321171+0000", 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1718000000000002e-05 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321223+0000", 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8986999999999998e-05 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:01.321262+0000", 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.115567221 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87233 5.e 5:7fc2a908:::benchmark_data_smithi082_106363_object87232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 439.63939770600001, 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.111294265, 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.943 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87233 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267937+0000", 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267935+0000", 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2663000000000001e-05 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267977+0000", 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5570000000000001e-06 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267987+0000", 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5379999999999996e-06 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.267991+0000", 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2032000000000003e-05 2023-10-09T04:35:35.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268073+0000", 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000133812 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268207+0000", 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012234499999999999 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.268330+0000", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0025285910000000002 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.270858+0000", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003702388 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.274561+0000", 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10452908399999999 2023-10-09T04:35:35.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379090+0000", 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7010999999999997e-05 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379187+0000", 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4277999999999999e-05 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.379231+0000", 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.111294265 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:87250 5.e 5:7cc8be50:::benchmark_data_smithi082_106363_object87249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 439.61980461899998, 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12868468, 2023-10-09T04:35:35.950 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87250 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287530+0000", 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287527+0000", 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3634999999999996e-05 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287591+0000", 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6143000000000001e-05 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287607+0000", 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1359999999999999e-06 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287614+0000", 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0812e-05 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287675+0000", 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000141256 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.287816+0000", 2023-10-09T04:35:35.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000197651 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.288014+0000", 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031527289999999999 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.291167+0000", 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001236439 2023-10-09T04:35:35.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.292403+0000", 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.123718986 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416122+0000", 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4619e-05 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416177+0000", 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7647000000000001e-05 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:28:16.416214+0000", 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12868468 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22573 6.e 6:793e1913:::benchmark_data_smithi082_128868_object22572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 196.886844236, 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23767322799999999, 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.958 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22573 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:35.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020488+0000", 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7503999999999998e-05 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020536+0000", 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4269000000000001e-05 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020560+0000", 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1192999999999999e-05 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020571+0000", 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016770700000000001 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020739+0000", 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000143914 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020883+0000", 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016347700000000001 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:35.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.021046+0000", 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022837199999999999 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023330+0000", 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015380669999999999 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.024868+0000", 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23309624000000001 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.257964+0000", 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011879699999999999 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258083+0000", 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0285999999999997e-05 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258163+0000", 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23767322799999999 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.965 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22581 6.e 6:754d3d1d:::benchmark_data_smithi082_128868_object22580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 196.88459895700001, 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23624266899999999, 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22581 2023-10-09T04:35:35.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022734+0000", 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3418999999999996e-05 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022808+0000", 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0966e-05 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022818+0000", 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1190000000000002e-06 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022825+0000", 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00054854399999999998 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023373+0000", 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000153327 2023-10-09T04:35:35.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023526+0000", 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020472700000000001 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023731+0000", 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001241736 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.024973+0000", 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00092022999999999996 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.025893+0000", 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.232957569 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258851+0000", 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0142999999999996e-05 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258921+0000", 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7206000000000001e-05 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258978+0000", 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23624266899999999 2023-10-09T04:35:35.972 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22589 6.e 6:78dafd59:::benchmark_data_smithi082_128868_object22588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 196.88119770500001, 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23663462099999999, 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.973 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22589 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026135+0000", 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7635999999999997e-05 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026173+0000", 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8362000000000001e-05 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026191+0000", 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7409999999999998e-06 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026200+0000", 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6614999999999999e-05 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026256+0000", 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000146713 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026403+0000", 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000186606 2023-10-09T04:35:35.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026590+0000", 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001099572 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.027689+0000", 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047519499999999998 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.028164+0000", 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.234474934 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262639+0000", 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5122000000000003e-05 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262714+0000", 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6870000000000003e-05 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262771+0000", 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23663462099999999 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22601 6.e 6:7da5b80c:::benchmark_data_smithi082_128868_object22600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 196.872548951, 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22826392000000001, 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.981 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22601 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:35.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034784+0000", 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6398999999999998e-05 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034861+0000", 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0020000000000006e-06 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034869+0000", 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.569e-06 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034872+0000", 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00035316999999999998 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035226+0000", 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000127016 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035353+0000", 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9709000000000004e-05 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035452+0000", 2023-10-09T04:35:35.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001076694 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.036529+0000", 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00034020799999999999 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.036869+0000", 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22608289600000001 2023-10-09T04:35:35.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262952+0000", 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4405000000000004e-05 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.263017+0000", 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2823000000000001e-05 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.263049+0000", 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22826392000000001 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25605 6.e 6:7b421b4b:::benchmark_data_smithi082_128868_object25604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 195.14310107700001, 2023-10-09T04:35:35.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.150442087, 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25605 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:35:35.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764232+0000", 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9565999999999999e-05 2023-10-09T04:35:35.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764282+0000", 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1229e-05 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764293+0000", 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9510000000000003e-06 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764299+0000", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00051598900000000001 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764815+0000", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000123836 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764939+0000", 2023-10-09T04:35:35.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000120155 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.765059+0000", 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034649339999999998 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.768524+0000", 2023-10-09T04:35:35.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8259999999999999e-06 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.768533+0000", 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14597078099999999 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914503+0000", 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010176800000000001 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914605+0000", 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0284999999999995e-05 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914675+0000", 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.150442087 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:35.995 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25625 6.e 6:7b55b5f9:::benchmark_data_smithi082_128868_object25624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 195.13597059599999, 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14644722900000001, 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:35.996 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25625 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:35:35.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771362+0000", 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4214000000000001e-05 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771407+0000", 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8134e-05 2023-10-09T04:35:35.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771425+0000", 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8423999999999998e-05 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771443+0000", 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016804999999999999 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:35.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771611+0000", 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014548400000000001 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771757+0000", 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018745599999999999 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:36.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771944+0000", 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016346850000000001 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.773579+0000", 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001057952 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.774637+0000", 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14293971699999999 2023-10-09T04:35:36.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917577+0000", 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000105895 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917682+0000", 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000128558 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917811+0000", 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14644722900000001 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25638 6.e 6:7908a266:::benchmark_data_smithi082_128868_object25637:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 195.12794693399999, 2023-10-09T04:35:36.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13866558900000001, 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25638 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:35:36.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779386+0000", 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2234000000000002e-05 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779439+0000", 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2211e-05 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779451+0000", 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3349999999999997e-06 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779456+0000", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0542e-05 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779487+0000", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1191000000000002e-05 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779578+0000", 2023-10-09T04:35:36.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111891 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779690+0000", 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0012906339999999999 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.780980+0000", 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00065067399999999996 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.781631+0000", 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13629439099999999 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917925+0000", 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2866000000000003e-05 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917998+0000", 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4775000000000003e-05 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918053+0000", 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13866558900000001 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.010 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:26828 6.e 6:7f2ece57:::benchmark_data_smithi082_128868_object26827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 194.40323763200001, 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13063107199999999, 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 26828 2023-10-09T04:35:36.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:36.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504095+0000", 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8353999999999999e-05 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504133+0000", 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4508000000000001e-05 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504148+0000", 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0929999999999999e-06 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504154+0000", 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4948999999999999e-05 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504209+0000", 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000126699 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504335+0000", 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017422600000000001 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504510+0000", 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001165529 2023-10-09T04:35:36.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.505675+0000", 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026620700000000002 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.505941+0000", 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12852864 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634470+0000", 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000151111 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634621+0000", 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000106601 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634728+0000", 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13063107199999999 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:31791 6.e 6:7488fad2:::benchmark_data_smithi082_128868_object31790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 191.51601495400001, 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11084435099999999, 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.018 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 31791 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391318+0000", 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3900000000000003e-05 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391362+0000", 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1357e-05 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391373+0000", 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391379+0000", 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0275000000000001e-05 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391440+0000", 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000163346 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391603+0000", 2023-10-09T04:35:36.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000213604 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391817+0000", 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0019215269999999999 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393738+0000", 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3773000000000003e-05 2023-10-09T04:35:36.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393782+0000", 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10828449900000001 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502066+0000", 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7272999999999999e-05 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502124+0000", 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.019e-05 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502164+0000", 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11084435099999999 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:31792 6.e 6:79f14372:::benchmark_data_smithi082_128868_object31791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 191.515825616, 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11087027300000001, 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.026 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 31792 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391508+0000", 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3638999999999999e-05 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391561+0000", 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1829999999999994e-06 2023-10-09T04:35:36.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391569+0000", 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.442e-06 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391574+0000", 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001329513 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.392903+0000", 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2236000000000003e-05 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.392986+0000", 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112772 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:35:36.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393098+0000", 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00078891900000000001 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393887+0000", 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00044481099999999998 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.394332+0000", 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.107957769 2023-10-09T04:35:36.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502290+0000", 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3843e-05 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502334+0000", 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5241999999999998e-05 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502379+0000", 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11087027300000001 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:79283 6.7 6:efa62da7:::benchmark_data_smithi082_128868_object79282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 154.31823756700001, 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11057702599999999, 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.033 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 79283 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589095+0000", 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7123e-05 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589173+0000", 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7784000000000001e-05 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589190+0000", 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3559999999999993e-06 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589199+0000", 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00173497 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.590934+0000", 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112447 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.591046+0000", 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018018299999999999 2023-10-09T04:35:36.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.591226+0000", 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003313758 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.594540+0000", 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10471435 2023-10-09T04:35:36.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699254+0000", 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000150841 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699405+0000", 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026862300000000001 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699674+0000", 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11057702599999999 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:79289 6.6 6:6660557e:::benchmark_data_smithi082_128868_object79288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:35:36.040 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 154.31082446799999, 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114452888, 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 79289 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596508+0000", 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2818000000000006e-05 2023-10-09T04:35:36.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596591+0000", 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4948e-05 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596606+0000", 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4660000000000002e-06 2023-10-09T04:35:36.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596613+0000", 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3134999999999999e-05 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596666+0000", 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9699999999999998e-05 2023-10-09T04:35:36.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596756+0000", 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000101748 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596858+0000", 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027267839999999999 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.599584+0000", 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000327559 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.599912+0000", 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.110935303 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710847+0000", 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7570999999999994e-05 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710915+0000", 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7902999999999999e-05 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710963+0000", 2023-10-09T04:35:36.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114452888 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98683 6.6 6:664c4778:::benchmark_data_smithi082_128868_object11345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 73.046523128000004, 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23657092900000001, 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.048 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98683 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860810+0000", 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2110000000000001e-06 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860817+0000", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.38e-05 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860831+0000", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2699999999999999e-06 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860838+0000", 2023-10-09T04:35:36.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0358999999999998e-05 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860878+0000", 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010762100000000001 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860986+0000", 2023-10-09T04:35:36.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5366999999999998e-05 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861071+0000", 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0053058259999999996 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.866377+0000", 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049759499999999998 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.871353+0000", 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22573111000000001 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097084+0000", 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00023964599999999999 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097324+0000", 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8329999999999999e-05 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097382+0000", 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23657092900000001 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.055 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98684 6.6 6:69085b6d:::benchmark_data_smithi082_128868_object11346:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 73.046387010000004, 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23656778000000001, 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98684 2023-10-09T04:35:36.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:35:36.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860946+0000", 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8239999999999996e-06 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860952+0000", 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4110000000000002e-06 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860962+0000", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6619999999999999e-06 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860968+0000", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026980799999999998 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863666+0000", 2023-10-09T04:35:36.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012220800000000001 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863789+0000", 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014059200000000001 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863929+0000", 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002663117 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.866592+0000", 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049010529999999998 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.871493+0000", 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22593812799999999 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097431+0000", 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4642999999999999e-05 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097476+0000", 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9042000000000003e-05 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:35:36.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097515+0000", 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23656778000000001 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:35:36.063 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:35:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:36.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:36.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:35 smithi153 bash[59565]: cluster 2023-10-09T04:35:34.378878+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v618: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24530/88542 objects degraded (27.704%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:36.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:35 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.597378+0000 mon.a (mon.0) 950 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:36.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:35 smithi153 bash[59565]: audit 2023-10-09T04:35:35.606883+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:36.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:35 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.607016+0000 mon.a (mon.0) 952 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:35:36.157 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:35 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.610141+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:36.165 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:36.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:36.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[77063]: cluster 2023-10-09T04:35:34.378878+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v618: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24530/88542 objects degraded (27.704%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:36.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.597378+0000 mon.a (mon.0) 950 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[77063]: audit 2023-10-09T04:35:35.606883+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.607016+0000 mon.a (mon.0) 952 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.610141+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[79733]: cluster 2023-10-09T04:35:34.378878+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v618: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 24530/88542 objects degraded (27.704%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.597378+0000 mon.a (mon.0) 950 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[79733]: audit 2023-10-09T04:35:35.606883+0000 mon.a (mon.0) 951 : audit 1 from='client.? 172.21.15.82:0/37085349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.607016+0000 mon.a (mon.0) 952 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:35:36.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:35 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.610141+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:36.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:36.323 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:36.430 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:36.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:36.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:36.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:36.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:36.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:36.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:36.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:36.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:36.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:36.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:36.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:35:37.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:37.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:36 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.608367+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:36 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.609281+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:36 smithi153 bash[59565]: cluster 2023-10-09T04:35:35.611487+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:37.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.608367+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.609281+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[77063]: cluster 2023-10-09T04:35:35.611487+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.608367+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.609281+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:36 smithi082 bash[79733]: cluster 2023-10-09T04:35:35.611487+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T04:35:37.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:37.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:37.424 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:37.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:37.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:37.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:37.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:37.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:37.957 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:37.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:37.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:38.071 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:38.072 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:38.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:38.114 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:37 smithi082 bash[77063]: cluster 2023-10-09T04:35:36.379388+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v620: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24530/83775 objects degraded (29.281%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:35:38.114 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:37 smithi082 bash[79733]: cluster 2023-10-09T04:35:36.379388+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v620: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24530/83775 objects degraded (29.281%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:35:38.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:37 smithi153 bash[59565]: cluster 2023-10-09T04:35:36.379388+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v620: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 24530/83775 objects degraded (29.281%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:35:38.246 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:38.276 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:38.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:38.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:38.416 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:38.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:38.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:38.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:38.698 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:38.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:38.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:38.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:38.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:38.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:39.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:39.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:39.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:38 smithi153 bash[59565]: cluster 2023-10-09T04:35:38.380213+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v621: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24385/79647 objects degraded (30.616%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:35:39.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:39.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:38 smithi082 bash[77063]: cluster 2023-10-09T04:35:38.380213+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v621: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24385/79647 objects degraded (30.616%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:35:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:38 smithi082 bash[79733]: cluster 2023-10-09T04:35:38.380213+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v621: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 24385/79647 objects degraded (30.616%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:35:39.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:39.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:35:39.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:39.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:39.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:39.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:39.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:39.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:39.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:39.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:39.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:39 smithi082 bash[77063]: cluster 2023-10-09T04:35:39.591793+0000 mon.a (mon.0) 953 : cluster 3 Health check update: Degraded data redundancy: 24385/79647 objects degraded (30.616%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:39.918 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:39] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:35:39.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:39.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:39.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:40.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:40.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:39 smithi153 bash[59565]: cluster 2023-10-09T04:35:39.591793+0000 mon.a (mon.0) 953 : cluster 3 Health check update: Degraded data redundancy: 24385/79647 objects degraded (30.616%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:40.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:40.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:39 smithi082 bash[79733]: cluster 2023-10-09T04:35:39.591793+0000 mon.a (mon.0) 953 : cluster 3 Health check update: Degraded data redundancy: 24385/79647 objects degraded (30.616%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:40.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:40.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:40.338 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:40.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:40.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:40.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:40.623 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:40.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:40.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:40.794 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:40.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:40.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:40.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:40.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:41.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:41.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:41.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:41.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:40 smithi153 bash[59565]: cluster 2023-10-09T04:35:40.381007+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v622: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:41.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:41.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:40 smithi082 bash[77063]: cluster 2023-10-09T04:35:40.381007+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v622: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:41.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:40 smithi082 bash[79733]: cluster 2023-10-09T04:35:40.381007+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v622: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:41.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:41.350 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:41.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:41.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:41.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:41.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:41.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:41.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:41.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:41.847 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:41.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:41.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:41.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:41.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:42.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:42.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:42.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:42.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:42.297 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:42.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:42.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:42.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:42.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:42.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:42.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:42.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:42.854 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:42.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:42.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:42.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:42.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.004 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:43.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.140 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:43.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:43.274 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:43.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:43.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:43.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:43.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:43.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:43.648 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:43 smithi153 bash[59565]: cluster 2023-10-09T04:35:42.381581+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v623: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:43.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:43.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:43.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:43 smithi082 bash[77063]: cluster 2023-10-09T04:35:42.381581+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v623: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:43.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:43 smithi082 bash[79733]: cluster 2023-10-09T04:35:42.381581+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v623: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24385/74436 objects degraded (32.760%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:35:43.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:43.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:43.903 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:43.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:43.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:44.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:44.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:44.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:44.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:44.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:44.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:44.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:44.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:44.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:44.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:44.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:44.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:44.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:44.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:44 smithi153 bash[59565]: audit 2023-10-09T04:35:44.268722+0000 mon.a (mon.0) 954 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:44.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:44.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:44 smithi082 bash[79733]: audit 2023-10-09T04:35:44.268722+0000 mon.a (mon.0) 954 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:44 smithi082 bash[77063]: audit 2023-10-09T04:35:44.268722+0000 mon.a (mon.0) 954 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:44.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:44.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:44.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:45.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:45.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:45.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:45.242 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:45.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:45.416 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:45.458 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:45 smithi082 bash[77063]: cluster 2023-10-09T04:35:44.382398+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v624: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24225/69888 objects degraded (34.663%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:45.458 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:45 smithi082 bash[77063]: cluster 2023-10-09T04:35:44.592794+0000 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 24225/69888 objects degraded (34.663%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:45.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:45 smithi082 bash[79733]: cluster 2023-10-09T04:35:44.382398+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v624: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24225/69888 objects degraded (34.663%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:45.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:45 smithi082 bash[79733]: cluster 2023-10-09T04:35:44.592794+0000 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 24225/69888 objects degraded (34.663%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:45.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:45.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:45.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:45.667 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:45 smithi153 bash[59565]: cluster 2023-10-09T04:35:44.382398+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v624: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24225/69888 objects degraded (34.663%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:45.667 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:45 smithi153 bash[59565]: cluster 2023-10-09T04:35:44.592794+0000 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 24225/69888 objects degraded (34.663%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:45.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:45.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:45.692 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:45.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:45.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:45.842 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:35:45.885 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:35:45.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:45.945 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:35:45.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:45.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:45.998 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:35:46.036 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:35:46.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.087 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:35:46.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:46.129 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:35:46.173 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:35:46.181 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.181 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.182 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:46.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:46.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:46.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:46.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:46.466 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:35:46.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:46.597 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:46.597 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:46.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.621 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:46 smithi153 bash[59565]: audit 2023-10-09T04:35:45.786434+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.82:0/1203314325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:46.621 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:46 smithi153 bash[59565]: audit 2023-10-09T04:35:46.162225+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:35:46.635 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:46 smithi082 bash[77063]: audit 2023-10-09T04:35:45.786434+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.82:0/1203314325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:46 smithi082 bash[77063]: audit 2023-10-09T04:35:46.162225+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:35:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:46 smithi082 bash[79733]: audit 2023-10-09T04:35:45.786434+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.82:0/1203314325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:46 smithi082 bash[79733]: audit 2023-10-09T04:35:46.162225+0000 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:35:46.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:46.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:46.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:46.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:46.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:46.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:47.022 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:47.049 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:47.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:47.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:47.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:47.307 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:47.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:47.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:47.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:35:47.553 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:47.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:47 smithi153 bash[59565]: cluster 2023-10-09T04:35:46.385773+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v625: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 24225/65250 objects degraded (37.126%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:35:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:47 smithi153 bash[59565]: cluster 2023-10-09T04:35:46.458444+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T04:35:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:47 smithi153 bash[59565]: audit 2023-10-09T04:35:46.461546+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:35:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:47 smithi153 bash[59565]: cluster 2023-10-09T04:35:46.461626+0000 mon.a (mon.0) 958 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:35:47.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:47.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[79733]: cluster 2023-10-09T04:35:46.385773+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v625: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 24225/65250 objects degraded (37.126%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:35:47.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[79733]: cluster 2023-10-09T04:35:46.458444+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[79733]: audit 2023-10-09T04:35:46.461546+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[79733]: cluster 2023-10-09T04:35:46.461626+0000 mon.a (mon.0) 958 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[77063]: cluster 2023-10-09T04:35:46.385773+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v625: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 24225/65250 objects degraded (37.126%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[77063]: cluster 2023-10-09T04:35:46.458444+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[77063]: audit 2023-10-09T04:35:46.461546+0000 mon.a (mon.0) 957 : audit 1 from='client.? 172.21.15.82:0/4110810360' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:35:47.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:47 smithi082 bash[77063]: cluster 2023-10-09T04:35:46.461626+0000 mon.a (mon.0) 958 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:35:47.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:47.853 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:47.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:47.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:47.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:47.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:48.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:48.126 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:48.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:48.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:48.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:48.409 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:48.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:48.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:48.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:48.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:48.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:48.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:48.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:48.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:48.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:48.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:48.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:48.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:48.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:49.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:49.090 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:49.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:49.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:49.340 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:49.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:49.473 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:49 smithi082 bash[77063]: cluster 2023-10-09T04:35:48.386409+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v627: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 24080/61287 objects degraded (39.291%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:49.473 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:49 smithi082 bash[79733]: cluster 2023-10-09T04:35:48.386409+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v627: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 24080/61287 objects degraded (39.291%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:49.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:49.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:49.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:49.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:49 smithi153 bash[59565]: cluster 2023-10-09T04:35:48.386409+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v627: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 24080/61287 objects degraded (39.291%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:49.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:35:49.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:49] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:35:49.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:49.763 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:49.866 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:49.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:49.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:50.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:50.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:50.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:50.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:50.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:50.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:50.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:50.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:50.456 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:50.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:50 smithi082 bash[77063]: cluster 2023-10-09T04:35:49.594157+0000 mon.a (mon.0) 959 : cluster 3 Health check update: Degraded data redundancy: 24080/61287 objects degraded (39.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:50.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:50.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:50 smithi153 bash[59565]: cluster 2023-10-09T04:35:49.594157+0000 mon.a (mon.0) 959 : cluster 3 Health check update: Degraded data redundancy: 24080/61287 objects degraded (39.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:50.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:50.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:50 smithi082 bash[79733]: cluster 2023-10-09T04:35:49.594157+0000 mon.a (mon.0) 959 : cluster 3 Health check update: Degraded data redundancy: 24080/61287 objects degraded (39.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:50.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:50.840 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:50.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:50.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:50.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:51.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:51.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:51.155 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:51.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:51.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:51.287 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:51.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:51.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:51.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:51.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:51.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:51.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:51.698 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:51.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:51.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:51 smithi153 bash[59565]: cluster 2023-10-09T04:35:50.387126+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v628: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:51.938 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:51.952 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:51 smithi082 bash[77063]: cluster 2023-10-09T04:35:50.387126+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v628: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:51.952 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:51 smithi082 bash[79733]: cluster 2023-10-09T04:35:50.387126+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v628: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:51.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:51.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:52.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:52.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:52.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:52.369 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:52.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.409 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:52.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:52.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:52.679 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:52.784 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:52.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:52.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:52.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:52.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:53.060 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:53.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:53.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:53.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:53.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:53.369 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:53.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:53.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:53.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:53.621 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:53 smithi153 bash[59565]: cluster 2023-10-09T04:35:52.387535+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v629: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:53.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:53 smithi153 bash[59565]: audit 2023-10-09T04:35:53.409851+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.82:0/1417307592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:53 smithi153 bash[59565]: audit 2023-10-09T04:35:53.411548+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.625 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:35:53.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:53.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[77063]: cluster 2023-10-09T04:35:52.387535+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v629: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[77063]: audit 2023-10-09T04:35:53.409851+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.82:0/1417307592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[77063]: audit 2023-10-09T04:35:53.411548+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[79733]: cluster 2023-10-09T04:35:52.387535+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v629: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 24080/56226 objects degraded (42.827%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[79733]: audit 2023-10-09T04:35:53.409851+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.82:0/1417307592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.769 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:53 smithi082 bash[79733]: audit 2023-10-09T04:35:53.411548+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:53.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:53.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:53.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:53.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:54.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:54.442 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:54.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:54.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:54.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:54.635 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.605397+0000 mon.a (mon.0) 961 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:54.635 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.608363+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.635 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.612505+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: audit 2023-10-09T04:35:53.613478+0000 mon.a (mon.0) 962 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.613557+0000 mon.a (mon.0) 963 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.617276+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.622847+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.624363+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.629803+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.630171+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:53.631093+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.636 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[77063]: cluster 2023-10-09T04:35:54.595537+0000 mon.a (mon.0) 964 : cluster 3 Health check update: Degraded data redundancy: 24080/56226 objects degraded (42.827%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:54.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:54.742 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:54.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:54.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.605397+0000 mon.a (mon.0) 961 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.608363+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.612505+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: audit 2023-10-09T04:35:53.613478+0000 mon.a (mon.0) 962 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.613557+0000 mon.a (mon.0) 963 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.617276+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.622847+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.624363+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.629803+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.630171+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:53.631093+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:54 smithi153 bash[59565]: cluster 2023-10-09T04:35:54.595537+0000 mon.a (mon.0) 964 : cluster 3 Health check update: Degraded data redundancy: 24080/56226 objects degraded (42.827%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:54.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.605397+0000 mon.a (mon.0) 961 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.608363+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.612505+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: audit 2023-10-09T04:35:53.613478+0000 mon.a (mon.0) 962 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.613557+0000 mon.a (mon.0) 963 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:35:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.617276+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.622847+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.624363+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.629803+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.630171+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:53.631093+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T04:35:54.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:54 smithi082 bash[79733]: cluster 2023-10-09T04:35:54.595537+0000 mon.a (mon.0) 964 : cluster 3 Health check update: Degraded data redundancy: 24080/56226 objects degraded (42.827%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:55.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:55.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:55.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:55.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:55.273 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:55.381 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:55.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:55.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:55.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:55.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:55.661 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:55.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:55.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:55.830 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:55 smithi082 bash[77063]: cluster 2023-10-09T04:35:54.389744+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v631: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23930/51393 objects degraded (46.563%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:35:55.830 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:55 smithi082 bash[79733]: cluster 2023-10-09T04:35:54.389744+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v631: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23930/51393 objects degraded (46.563%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:35:55.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:55.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:55 smithi153 bash[59565]: cluster 2023-10-09T04:35:54.389744+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v631: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23930/51393 objects degraded (46.563%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:35:55.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:55.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:55.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:56.097 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:56.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:56.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:56.255 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:56.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:56.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:56.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:56.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:56.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:56.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:56.745 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:56.745 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:56.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:56.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:56.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:56.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:56.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:57.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:57.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:57.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:57.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:57.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:35:57.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:57.333 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:57.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:57.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:57.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:57.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:57.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:57.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:35:57.710 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:57.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:57.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:57.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:57.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:57.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:57 smithi082 bash[77063]: cluster 2023-10-09T04:35:56.390512+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v632: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 975 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 23930/46662 objects degraded (51.284%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:35:57.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:57 smithi082 bash[79733]: cluster 2023-10-09T04:35:56.390512+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v632: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 975 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 23930/46662 objects degraded (51.284%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:35:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:57 smithi153 bash[59565]: cluster 2023-10-09T04:35:56.390512+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v632: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 975 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 23930/46662 objects degraded (51.284%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:35:57.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:57.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:58.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:58.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:58.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:58.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:58.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:58.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:58.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:58.446 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:58.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:58.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:58.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:58.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:58.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:58.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:58.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:58.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:58.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:59.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:59.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:59.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:59.169 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:59.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:59.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:59.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:59.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:59.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:59.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:59.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:59.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:35:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:35:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:59 smithi153 bash[59565]: cluster 2023-10-09T04:35:58.392866+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v633: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 892 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/42666 objects degraded (55.728%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:59 smithi153 bash[59565]: audit 2023-10-09T04:35:59.268464+0000 mon.a (mon.0) 965 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:35:59 smithi153 bash[59565]: cluster 2023-10-09T04:35:59.596745+0000 mon.a (mon.0) 966 : cluster 3 Health check update: Degraded data redundancy: 23777/42666 objects degraded (55.728%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:59.957 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[77063]: cluster 2023-10-09T04:35:58.392866+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v633: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 892 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/42666 objects degraded (55.728%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:59.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[77063]: audit 2023-10-09T04:35:59.268464+0000 mon.a (mon.0) 965 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:59.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[77063]: cluster 2023-10-09T04:35:59.596745+0000 mon.a (mon.0) 966 : cluster 3 Health check update: Degraded data redundancy: 23777/42666 objects degraded (55.728%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:59.969 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:35:59] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:35:59.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[79733]: cluster 2023-10-09T04:35:58.392866+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v633: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 892 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/42666 objects degraded (55.728%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:35:59.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[79733]: audit 2023-10-09T04:35:59.268464+0000 mon.a (mon.0) 965 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:59.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:35:59 smithi082 bash[79733]: cluster 2023-10-09T04:35:59.596745+0000 mon.a (mon.0) 966 : cluster 3 Health check update: Degraded data redundancy: 23777/42666 objects degraded (55.728%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:59.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:35:59.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:00.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.115 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:00.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:00.246 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:00.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:00.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:00.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:00.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:00.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:00.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:00.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:00.924 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:00.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:01.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:01.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:01.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:01.213 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:01.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:01.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:01.450 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:01.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:01.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:01.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:01.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:01.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:01.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:01 smithi082 bash[77063]: cluster 2023-10-09T04:36:00.393618+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v634: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:01.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:01 smithi082 bash[79733]: cluster 2023-10-09T04:36:00.393618+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v634: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:01.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:01.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:01.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:01.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:01.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:01 smithi153 bash[59565]: cluster 2023-10-09T04:36:00.393618+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v634: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:01.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.017 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:02.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:02.155 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:02.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:02.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:02.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:02.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:02.532 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.559 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:02.562 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:02.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:02.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:02.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:02.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:02.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:02.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:02.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:02.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:03.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:03.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:03.229 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:03.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:03.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:03.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.532 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:03.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:03.671 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:36:03.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:03.673 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[77063]: cluster 2023-10-09T04:36:02.394142+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v635: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:03.673 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[77063]: audit 2023-10-09T04:36:03.028363+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.82:0/292535304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:03.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:03.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:03.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:03 smithi153 bash[59565]: cluster 2023-10-09T04:36:02.394142+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v635: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:03.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:03 smithi153 bash[59565]: audit 2023-10-09T04:36:03.028363+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.82:0/292535304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:03 smithi153 bash[59565]: audit 2023-10-09T04:36:03.029310+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.939 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[77063]: audit 2023-10-09T04:36:03.029310+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.940 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[79733]: cluster 2023-10-09T04:36:02.394142+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v635: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 23777/37596 objects degraded (63.243%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:36:03.940 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[79733]: audit 2023-10-09T04:36:03.028363+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.82:0/292535304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.940 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:03 smithi082 bash[79733]: audit 2023-10-09T04:36:03.029310+0000 mon.a (mon.0) 967 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:36:03.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:03.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:04.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:04.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:04.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:04.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:04.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:04.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:04.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:04.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:04.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:04.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:04.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:04.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:04.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:04.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:04.862 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: cluster 2023-10-09T04:36:03.659998+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.862 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: audit 2023-10-09T04:36:03.663229+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:36:04.862 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: cluster 2023-10-09T04:36:03.663335+0000 mon.a (mon.0) 969 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:36:04.862 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: cluster 2023-10-09T04:36:03.664204+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.863 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: cluster 2023-10-09T04:36:03.679477+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.863 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:04 smithi153 bash[59565]: cluster 2023-10-09T04:36:04.600353+0000 mon.a (mon.0) 970 : cluster 3 Health check update: Degraded data redundancy: 23777/37596 objects degraded (63.243%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:04.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:04.877 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: cluster 2023-10-09T04:36:03.659998+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.877 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: audit 2023-10-09T04:36:03.663229+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:36:04.878 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: cluster 2023-10-09T04:36:03.663335+0000 mon.a (mon.0) 969 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:36:04.878 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: cluster 2023-10-09T04:36:03.664204+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.878 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: cluster 2023-10-09T04:36:03.679477+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.878 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[77063]: cluster 2023-10-09T04:36:04.600353+0000 mon.a (mon.0) 970 : cluster 3 Health check update: Degraded data redundancy: 23777/37596 objects degraded (63.243%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:04.878 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: cluster 2023-10-09T04:36:03.659998+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: audit 2023-10-09T04:36:03.663229+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:36:04.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: cluster 2023-10-09T04:36:03.663335+0000 mon.a (mon.0) 969 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:36:04.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: cluster 2023-10-09T04:36:03.664204+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: cluster 2023-10-09T04:36:03.679477+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T04:36:04.879 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:04 smithi082 bash[79733]: cluster 2023-10-09T04:36:04.600353+0000 mon.a (mon.0) 970 : cluster 3 Health check update: Degraded data redundancy: 23777/37596 objects degraded (63.243%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:04.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:04.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:04.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:05.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:05.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:05.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:05.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:05.302 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:05.406 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:05.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:05.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:05.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:05.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:05.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:05.694 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:05.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:05.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:05.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:05.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:05.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:05 smithi082 bash[77063]: cluster 2023-10-09T04:36:04.394745+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v637: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 689 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/32916 objects degraded (71.701%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:05 smithi082 bash[79733]: cluster 2023-10-09T04:36:04.394745+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v637: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 689 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/32916 objects degraded (71.701%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:06.016 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:06.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:06.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:06.156 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:05 smithi153 bash[59565]: cluster 2023-10-09T04:36:04.394745+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v637: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 689 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/32916 objects degraded (71.701%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:06.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:06.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:36:06.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:06.307 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:06.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:06.544 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:06.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:06.575 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:06.657 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:36:06.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.703 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:36:06.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:06.735 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:36:06.777 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:36:06.811 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:36:06.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.842 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:36:06.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:06.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:06.873 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:36:06.912 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:36:06.914 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.914 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.914 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.914 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.914 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.915 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.915 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.915 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:36:06.946 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:06 smithi082 bash[77063]: audit 2023-10-09T04:36:06.588634+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.82:0/2798339406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:06.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:06 smithi082 bash[79733]: audit 2023-10-09T04:36:06.588634+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.82:0/2798339406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:06.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:06.961 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:06 smithi153 bash[59565]: audit 2023-10-09T04:36:06.588634+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.82:0/2798339406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:06.993 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:36:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:36:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:36:06.993 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34794.0:38624 5.4 5:207acc77:::benchmark_data_smithi082_106363_object38623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e110)", 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 512.17763507799998, 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183254056, 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34794", 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3801426066", 2023-10-09T04:36:06.994 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 38624 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807564+0000", 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:06.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807562+0000", 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0506999999999997e-05 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807602+0000", 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7394e-05 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.996 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807620+0000", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7710000000000006e-06 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807628+0000", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9972999999999997e-05 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807677+0000", 2023-10-09T04:36:06.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111838 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807789+0000", 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100987 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.807890+0000", 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00161356 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.809504+0000", 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023769960000000001 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.811881+0000", 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.178780785 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:06.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990662+0000", 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0154000000000005e-05 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990752+0000", 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5858999999999994e-05 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:27:34.990818+0000", 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183254056 2023-10-09T04:36:07.000 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22579 6.8 6:189a18fb:::benchmark_data_smithi082_128868_object22578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.022620+0000", 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 227.96257897500001, 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23223287100000001, 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.001 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22579 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022620+0000", 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022620+0000", 2023-10-09T04:36:07.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022618+0000", 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8961000000000001e-05 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022667+0000", 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0101e-05 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022677+0000", 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9599999999999999e-06 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022682+0000", 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9610000000000001e-05 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022731+0000", 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014852100000000001 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022880+0000", 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018233600000000001 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023062+0000", 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015072009999999999 2023-10-09T04:36:07.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.024570+0000", 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001327143 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.025897+0000", 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22851988100000001 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.254417+0000", 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000175267 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.254592+0000", 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026077799999999998 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.254853+0000", 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23223287100000001 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22588 6.3 6:c4c0e6d9:::benchmark_data_smithi082_128868_object22587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.026022+0000", 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 227.95917621199999, 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23272599699999999, 2023-10-09T04:36:07.008 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22588 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026022+0000", 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026022+0000", 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026021+0000", 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5219000000000002e-05 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026066+0000", 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3925e-05 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026080+0000", 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3039999999999999e-06 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026087+0000", 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3917000000000005e-05 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026151+0000", 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015707399999999999 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026308+0000", 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016409500000000001 2023-10-09T04:36:07.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026473+0000", 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001010653 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.027483+0000", 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0011782240000000001 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.028661+0000", 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22992637699999999 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258588+0000", 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4707000000000001e-05 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258673+0000", 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5970000000000006e-05 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258748+0000", 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23272599699999999 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22594 6.8 6:19485a06:::benchmark_data_smithi082_128868_object22593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.029187+0000", 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 227.95601175300001, 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.229784497, 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22594 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.016 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029187+0000", 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029187+0000", 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029186+0000", 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9035e-05 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029245+0000", 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2904999999999999e-05 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029258+0000", 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8839999999999999e-06 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029264+0000", 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4332999999999998e-05 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029298+0000", 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013113 2023-10-09T04:36:07.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029429+0000", 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000176651 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.029606+0000", 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001249198 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.030855+0000", 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00028423100000000002 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.031139+0000", 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22762728300000001 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258766+0000", 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011897500000000001 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258885+0000", 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6133999999999999e-05 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258971+0000", 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.229784497 2023-10-09T04:36:07.022 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25620 6.b 6:dc31e5fc:::benchmark_data_smithi082_128868_object25619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.770777+0000", 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 226.21442172100001, 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147406589, 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.023 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25620 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770777+0000", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770777+0000", 2023-10-09T04:36:07.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770776+0000", 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5898000000000001e-05 2023-10-09T04:36:07.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770812+0000", 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0299999999999994e-06 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770820+0000", 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4700000000000008e-06 2023-10-09T04:36:07.026 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770829+0000", 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3130999999999997e-05 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770882+0000", 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000144454 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771027+0000", 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000188891 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771216+0000", 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033167159999999999 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.774532+0000", 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2706e-05 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.774545+0000", 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14335799099999999 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917903+0000", 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000155926 2023-10-09T04:36:07.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918059+0000", 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000124518 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918184+0000", 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147406589 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.030 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25622 6.b 6:de213153:::benchmark_data_smithi082_128868_object25621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.770840+0000", 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 226.21435834299999, 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147862617, 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.031 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25622 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770840+0000", 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770840+0000", 2023-10-09T04:36:07.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770840+0000", 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.3435000000000001e-05 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770873+0000", 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5839999999999995e-06 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770882+0000", 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5990000000000002e-06 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.770885+0000", 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024358100000000001 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.773321+0000", 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014195399999999999 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.773463+0000", 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011881 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.773582+0000", 2023-10-09T04:36:07.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0021413030000000002 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.775723+0000", 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000134105 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.775857+0000", 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.142563038 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918420+0000", 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000163721 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918584+0000", 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000119011 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918703+0000", 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147862617 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:26812 6.3 6:c83ffda3:::benchmark_data_smithi082_128868_object26811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:21.497348+0000", 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 225.487850382, 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13900278799999999, 2023-10-09T04:36:07.038 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 26812 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497348+0000", 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497348+0000", 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497347+0000", 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8118999999999998e-05 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497395+0000", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7020000000000001e-05 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497432+0000", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5682000000000002e-05 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497468+0000", 2023-10-09T04:36:07.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3467999999999996e-05 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497531+0000", 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7219000000000006e-05 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497618+0000", 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013101200000000001 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497749+0000", 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024000559999999998 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.500149+0000", 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0988e-05 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.500210+0000", 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13578628100000001 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.635997+0000", 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016152500000000001 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636158+0000", 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019297100000000001 2023-10-09T04:36:07.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636351+0000", 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13900278799999999 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:26813 6.3 6:cb02ec46:::benchmark_data_smithi082_128868_object26812:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:21.497507+0000", 2023-10-09T04:36:07.045 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 225.48769153699999, 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13924391, 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 26813 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497507+0000", 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497507+0000", 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497506+0000", 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2867999999999997e-05 2023-10-09T04:36:07.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497549+0000", 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2415000000000001e-05 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497562+0000", 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8067000000000002e-05 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.497580+0000", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001210481 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.498790+0000", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013036699999999999 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.498921+0000", 2023-10-09T04:36:07.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000128138 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.499049+0000", 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0013774289999999999 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.500426+0000", 2023-10-09T04:36:07.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00064734999999999999 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501073+0000", 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.135397721 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636471+0000", 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015582000000000001 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636627+0000", 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000124086 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636751+0000", 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13924391 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:26823 6.3 6:cbaa81c9:::benchmark_data_smithi082_128868_object26822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:21.501075+0000", 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 225.484123485, 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13602350899999999, 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.053 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 26823 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501075+0000", 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501075+0000", 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:07.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501074+0000", 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8436000000000004e-05 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501172+0000", 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.272e-05 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501185+0000", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.639e-06 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501190+0000", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5685000000000001e-05 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501225+0000", 2023-10-09T04:36:07.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000135294 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501361+0000", 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000157152 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.501518+0000", 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0021519410000000001 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.503670+0000", 2023-10-09T04:36:07.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00057795399999999999 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504248+0000", 2023-10-09T04:36:07.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13262533200000001 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636873+0000", 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000121072 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.636994+0000", 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000104561 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.637099+0000", 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13602350899999999 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:87212 6.5 6:a9473a54:::benchmark_data_smithi082_128868_object87211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:09.740358+0000", 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.24484045700001, 2023-10-09T04:36:07.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.022790074, 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87212 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740358+0000", 2023-10-09T04:36:07.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740358+0000", 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740356+0000", 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8814999999999998e-05 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740415+0000", 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6444e-05 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740431+0000", 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7839999999999998e-06 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740441+0000", 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000104956 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740546+0000", 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017901000000000001 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740725+0000", 2023-10-09T04:36:07.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020393399999999999 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.740929+0000", 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0036361229999999998 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.744565+0000", 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044264899999999999 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.748992+0000", 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0069229010000000004 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755914+0000", 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1160000000000005e-06 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:36:07.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755924+0000", 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0071213969999999 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763045+0000", 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103342 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763148+0000", 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.022790074 2023-10-09T04:36:07.068 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:87218 6.3 6:c648fe40:::benchmark_data_smithi082_128868_object87217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:09.749443+0000", 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.23575556399999, 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0339465830000001, 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.069 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87218 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749443+0000", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749443+0000", 2023-10-09T04:36:07.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749441+0000", 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7759e-05 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749499+0000", 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9347999999999998e-05 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749518+0000", 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0077e-05 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749528+0000", 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9888000000000006e-05 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.072 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749598+0000", 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2790000000000001e-06 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op must wait for map", 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.749606+0000", 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0054501699999999998 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755056+0000", 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9323000000000001e-05 2023-10-09T04:36:07.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755075+0000", 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.007713971 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.762789+0000", 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000173921 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.762963+0000", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015627499999999999 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763119+0000", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0036241989999999998 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.766743+0000", 2023-10-09T04:36:07.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.016546299 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.783290+0000", 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9610999999999997e-05 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.783329+0000", 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0418999999999999e-05 2023-10-09T04:36:07.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.783390+0000", 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0339465830000001 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:87221 6.5 6:a1c672c2:::benchmark_data_smithi082_128868_object87220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T04:36:07.077 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:09.755753+0000", 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.22944613799999, 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0114522260000001, 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 87221 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.078 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755753+0000", 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755753+0000", 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755751+0000", 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5049999999999997e-05 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755796+0000", 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2928999999999999e-05 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755809+0000", 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0413e-05 2023-10-09T04:36:07.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.755819+0000", 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00089787799999999996 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.756717+0000", 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3470000000000004e-06 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:09.756723+0000", 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0064666790000001 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763190+0000", 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000150047 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763340+0000", 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000184585 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.763525+0000", 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035203610000000001 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.767045+0000", 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.375e-05 2023-10-09T04:36:07.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.767089+0000", 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7292000000000001e-05 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.767126+0000", 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8823000000000001e-05 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:10.767205+0000", 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0114522260000001 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98680 6.5 6:ac6fab3a:::benchmark_data_smithi082_128868_object11342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.859611+0000", 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 104.125587974, 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.24580366200000001, 2023-10-09T04:36:07.085 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98680 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859611+0000", 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859611+0000", 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859609+0000", 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.494e-06 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859616+0000", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4179999999999997e-06 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859624+0000", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9289999999999997e-06 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859629+0000", 2023-10-09T04:36:07.088 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4657000000000003e-05 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859674+0000", 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3079999999999997e-05 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859767+0000", 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1403999999999994e-05 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.859858+0000", 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23642273799999999 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096281+0000", 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013476699999999999 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096416+0000", 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0088956139999999996 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.105311+0000", 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8047000000000002e-05 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.105370+0000", 2023-10-09T04:36:07.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4838e-05 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.105414+0000", 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.24580366200000001 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.092 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98681 6.8 6:1cd09604:::benchmark_data_smithi082_128868_object11343:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.860433+0000", 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 104.124765647, 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.24285289099999999, 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98681 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860433+0000", 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860433+0000", 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860432+0000", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9280000000000001e-06 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860437+0000", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.229150983 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089588+0000", 2023-10-09T04:36:07.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.809e-05 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089616+0000", 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7505999999999998e-05 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089714+0000", 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018551200000000001 2023-10-09T04:36:07.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089899+0000", 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00019670899999999999 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.090096+0000", 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0067433989999999997 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096839+0000", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3399999999999998e-06 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096848+0000", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0063719099999999997 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.103220+0000", 2023-10-09T04:36:07.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7169000000000003e-05 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.103257+0000", 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9198000000000001e-05 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.103286+0000", 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.24285289099999999 2023-10-09T04:36:07.099 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop_reply(client.34995.0:98680 6.5 e148/146 ondisk, result = 0)", 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.861744+0000", 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 104.123455064, 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.234542523, 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861744+0000", 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861744+0000", 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861743+0000", 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0589999999999999e-06 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861749+0000", 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22782177300000001 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089571+0000", 2023-10-09T04:36:07.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4876e-05 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.089595+0000", 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0066660549999999997 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096261+0000", 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6399999999999992e-06 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096271+0000", 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5079000000000001e-05 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.096286+0000", 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.234542523 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:152096 6.3 6:c0003b5e:::benchmark_data_smithi082_128868_object64758:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:42.576480+0000", 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 24.408719100999999, 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21203454599999999, 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.105 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 152096 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576480+0000", 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576480+0000", 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:07.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576479+0000", 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4900000000000002e-06 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576483+0000", 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1840000000000004e-06 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.107 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576490+0000", 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1849999999999997e-06 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.576494+0000", 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0046952670000000004 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.108 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.581189+0000", 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012244799999999999 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.581312+0000", 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013969000000000001 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.581451+0000", 2023-10-09T04:36:07.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0039894860000000004 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.585441+0000", 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0081371250000000003 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.593578+0000", 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19460923299999999 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.788187+0000", 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000199571 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.788387+0000", 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012753299999999999 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.788514+0000", 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21203454599999999 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:152098 6.5 6:ae97cdfb:::benchmark_data_smithi082_128868_object64760:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:42.582288+0000", 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 24.402910589000001, 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21834838000000001, 2023-10-09T04:36:07.112 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 152098 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582288+0000", 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.113 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582288+0000", 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582286+0000", 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8040000000000002e-06 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.114 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582294+0000", 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4001e-05 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582308+0000", 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4229999999999999e-06 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.115 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.582315+0000", 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001283101 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.583599+0000", 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00028671099999999999 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.583885+0000", 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000631417 2023-10-09T04:36:07.116 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.584517+0000", 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0041823340000000002 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.588699+0000", 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.008298761 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.117 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.596998+0000", 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20347087799999999 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800469+0000", 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011848799999999999 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.118 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800587+0000", 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9393e-05 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800636+0000", 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21834838000000001 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.119 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:152110 6.5 6:acbdfe2c:::benchmark_data_smithi082_128868_object64772:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:42.608901+0000", 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 24.376297352999998, 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19200704800000001, 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 152110 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.120 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608901+0000", 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608901+0000", 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.121 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608899+0000", 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.238e-05 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608911+0000", 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0270000000000001e-05 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608931+0000", 2023-10-09T04:36:07.122 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6139999999999992e-06 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608940+0000", 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2880999999999997e-05 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.608993+0000", 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010895099999999999 2023-10-09T04:36:07.123 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.609102+0000", 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112236 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.609214+0000", 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0037647409999999998 2023-10-09T04:36:07.124 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.612979+0000", 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011273059 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.624252+0000", 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.176562422 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.125 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800814+0000", 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3718000000000003e-05 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800868+0000", 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0395999999999997e-05 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.126 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.800908+0000", 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19200704800000001 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:152129 6.3 6:c94259e6:::benchmark_data_smithi082_128868_object64791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:42.665010+0000", 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 24.320189120999999, 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13492536599999999, 2023-10-09T04:36:07.127 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 152129 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665010+0000", 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.128 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665010+0000", 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665008+0000", 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4260000000000003e-06 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.129 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665016+0000", 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7795000000000002e-05 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665034+0000", 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0033e-05 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.130 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665044+0000", 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5173000000000003e-05 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665089+0000", 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000110875 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665200+0000", 2023-10-09T04:36:07.131 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000147498 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.665348+0000", 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0043965580000000001 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.669744+0000", 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027571509999999998 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.132 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.672501+0000", 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12733613199999999 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.799838+0000", 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3451e-05 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.133 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.799891+0000", 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3955999999999997e-05 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.799935+0000", 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13492536599999999 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.134 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:152132 6.5 6:a8a118a8:::benchmark_data_smithi082_128868_object64794:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:42.671529+0000", 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 24.313669580999999, 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.129630891, 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 152132 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.135 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671529+0000", 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671529+0000", 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.136 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671527+0000", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3536999999999999e-05 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671540+0000", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.8504999999999999e-05 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671569+0000", 2023-10-09T04:36:07.137 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7774e-05 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671586+0000", 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8303000000000001e-05 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671645+0000", 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012339 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.138 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671768+0000", 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012698200000000001 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.671895+0000", 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0052492509999999999 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:07.139 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.677144+0000", 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001779849 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.678924+0000", 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12212983400000001 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.801054+0000", 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8687999999999998e-05 2023-10-09T04:36:07.140 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.801113+0000", 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7302999999999998e-05 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:42.801160+0000", 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.129630891 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.141 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:07.142 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:07.142 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:36:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:07.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:07.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:07.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:07.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:07.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:07.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:07.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:07.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:07.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:07.799 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:07 smithi082 bash[77063]: cluster 2023-10-09T04:36:06.395475+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v638: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 590 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/28197 objects degraded (83.700%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:07.800 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:07 smithi082 bash[79733]: cluster 2023-10-09T04:36:06.395475+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v638: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 590 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/28197 objects degraded (83.700%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:07.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:07.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:07.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:07.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:07.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:07.963 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:07 smithi153 bash[59565]: cluster 2023-10-09T04:36:06.395475+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v638: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 590 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 23601/28197 objects degraded (83.700%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:36:07.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:08.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:36:08.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:08.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:08.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:08.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:08.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:08.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:08.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:08.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:08.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:08.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:08.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:08.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:08.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:08.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:08.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:08.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:09.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:09.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:09.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:09.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:09.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:09.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:09.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:09.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:09.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:09.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:09.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:09.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:09.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:09.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:09.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:09.911 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:09 smithi082 bash[77063]: cluster 2023-10-09T04:36:08.397517+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v639: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 508 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 23435/24261 objects degraded (96.595%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:09.911 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:09 smithi082 bash[77063]: cluster 2023-10-09T04:36:09.603611+0000 mon.a (mon.0) 971 : cluster 3 Health check update: Degraded data redundancy: 23435/24261 objects degraded (96.595%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:09.911 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:09 smithi082 bash[79733]: cluster 2023-10-09T04:36:08.397517+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v639: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 508 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 23435/24261 objects degraded (96.595%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:09.911 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:09 smithi082 bash[79733]: cluster 2023-10-09T04:36:09.603611+0000 mon.a (mon.0) 971 : cluster 3 Health check update: Degraded data redundancy: 23435/24261 objects degraded (96.595%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:09.911 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:09] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:36:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:09 smithi153 bash[59565]: cluster 2023-10-09T04:36:08.397517+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v639: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 508 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 23435/24261 objects degraded (96.595%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:09 smithi153 bash[59565]: cluster 2023-10-09T04:36:09.603611+0000 mon.a (mon.0) 971 : cluster 3 Health check update: Degraded data redundancy: 23435/24261 objects degraded (96.595%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:09.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:09.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:10.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:10.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:10.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:10.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:10.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:10.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:10.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:10.578 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:10.608 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:10.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:10.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:10.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:10.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:11.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:11.043 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:11.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:11.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:11.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:36:11.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:11.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:11.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:11.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:11.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:11.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:11.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:11.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:11.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:11.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:11.918 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:11 smithi153 bash[59565]: cluster 2023-10-09T04:36:10.398352+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v640: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:11.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:11.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:11.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:11.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:11 smithi082 bash[77063]: cluster 2023-10-09T04:36:10.398352+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v640: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:11.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:11 smithi082 bash[79733]: cluster 2023-10-09T04:36:10.398352+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v640: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:12.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:12.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:12.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:12.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:12.318 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:12.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:12.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:12.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:12.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:12.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:12.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:12.790 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:12.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:12.862 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:12.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:12.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:12.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:13.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:13.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:13.182 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.220 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:13.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:13.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:13.595 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:13.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.740 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:36:13.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:13.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:13.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:13.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:13.924 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[79733]: cluster 2023-10-09T04:36:12.399464+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v641: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[79733]: audit 2023-10-09T04:36:12.894010+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.82:0/721472641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[79733]: audit 2023-10-09T04:36:12.895013+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[77063]: cluster 2023-10-09T04:36:12.399464+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v641: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[77063]: audit 2023-10-09T04:36:12.894010+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.82:0/721472641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:13.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:13 smithi082 bash[77063]: audit 2023-10-09T04:36:12.895013+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:14.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:14.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:14.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:13 smithi153 bash[59565]: cluster 2023-10-09T04:36:12.399464+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v641: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 408 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23435/19470 objects degraded (120.365%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:13 smithi153 bash[59565]: audit 2023-10-09T04:36:12.894010+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.82:0/721472641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:13 smithi153 bash[59565]: audit 2023-10-09T04:36:12.895013+0000 mon.a (mon.0) 972 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:14.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:14.286 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:14.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:14.427 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:14.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:14.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:14.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:14.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:14.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:14.735 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:14.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:13.709993+0000 mon.a (mon.0) 973 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:14.823 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:13.723107+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: audit 2023-10-09T04:36:13.725593+0000 mon.a (mon.0) 974 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:13.725698+0000 mon.a (mon.0) 975 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:13.736358+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:13.744663+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: audit 2023-10-09T04:36:14.275694+0000 mon.a (mon.0) 976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: audit 2023-10-09T04:36:14.276910+0000 mon.a (mon.0) 977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.824 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:14 smithi153 bash[59565]: cluster 2023-10-09T04:36:14.604826+0000 mon.a (mon.0) 978 : cluster 3 Health check update: Degraded data redundancy: 23435/19470 objects degraded (120.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:14.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:14.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:14.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:14.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:13.709993+0000 mon.a (mon.0) 973 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:13.723107+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: audit 2023-10-09T04:36:13.725593+0000 mon.a (mon.0) 974 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:13.725698+0000 mon.a (mon.0) 975 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:36:14.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:13.736358+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:13.744663+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: audit 2023-10-09T04:36:14.275694+0000 mon.a (mon.0) 976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: audit 2023-10-09T04:36:14.276910+0000 mon.a (mon.0) 977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[77063]: cluster 2023-10-09T04:36:14.604826+0000 mon.a (mon.0) 978 : cluster 3 Health check update: Degraded data redundancy: 23435/19470 objects degraded (120.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:13.709993+0000 mon.a (mon.0) 973 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:13.723107+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: audit 2023-10-09T04:36:13.725593+0000 mon.a (mon.0) 974 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:13.725698+0000 mon.a (mon.0) 975 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:13.736358+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:13.744663+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: audit 2023-10-09T04:36:14.275694+0000 mon.a (mon.0) 976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: audit 2023-10-09T04:36:14.276910+0000 mon.a (mon.0) 977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:14 smithi082 bash[79733]: cluster 2023-10-09T04:36:14.604826+0000 mon.a (mon.0) 978 : cluster 3 Health check update: Degraded data redundancy: 23435/19470 objects degraded (120.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:15.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:15.115 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:15.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:15.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:15.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:15.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:15.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:15.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:15.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:15.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:15 smithi153 bash[59565]: cluster 2023-10-09T04:36:14.400076+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v643: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 310 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 23257/14748 objects degraded (157.696%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:15.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.835 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:15 smithi082 bash[77063]: cluster 2023-10-09T04:36:14.400076+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v643: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 310 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 23257/14748 objects degraded (157.696%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:15.835 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:15 smithi082 bash[79733]: cluster 2023-10-09T04:36:14.400076+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v643: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 310 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 23257/14748 objects degraded (157.696%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:15.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:15.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:16.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:16.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:16.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:16.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:16.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:16.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:16.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:16.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:16.614 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:16.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:16.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:16.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:16.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:16.876 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:16.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:16.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:16.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:173021 6.c 6:3796f8b4:::benchmark_data_smithi082_128868_object85683:head [delete] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:16.934075+0000", 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.042242899, 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.042278050999999997, 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:16.980 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 173021 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934075+0000", 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934075+0000", 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934073+0000", 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.222e-05 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934085+0000", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8912000000000001e-05 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934104+0000", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7159999999999998e-06 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.934114+0000", 2023-10-09T04:36:16.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026385530000000001 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.936752+0000", 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021461000000000001 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.936967+0000", 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024807699999999998 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937215+0000", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.006629299 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.943844+0000", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0055276149999999996 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.949372+0000", 2023-10-09T04:36:16.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.015296538 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:173023 6.c 6:3e5123a0:::benchmark_data_smithi082_128868_object85685:head [delete] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:16.937522+0000", 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.038796451000000003, 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.038928654, 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:36:16.986 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 173023 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937522+0000", 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937522+0000", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937520+0000", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9639999999999992e-06 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937529+0000", 2023-10-09T04:36:16.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5964000000000002e-05 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937545+0000", 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5309999999999998e-06 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.937552+0000", 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.006574908 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.944127+0000", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021455500000000001 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.944341+0000", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024420699999999997 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.944585+0000", 2023-10-09T04:36:16.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049608719999999999 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.949546+0000", 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9505000000000002e-05 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.949586+0000", 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.012063974999999999 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.991 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173040 6.a e161/151 6:55188d43:::benchmark_data_smithi082_128868_object85702:head v 161'11061, mlcod=138'5485)", 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:16.972165+0000", 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.004153599, 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0043672529999999998, 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:16.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972165+0000", 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972165+0000", 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972164+0000", 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4169999999999998e-06 2023-10-09T04:36:16.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972173+0000", 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3178999999999999e-05 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972196+0000", 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4570000000000005e-06 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972205+0000", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024054400000000001 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972445+0000", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0010000000000003e-05 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.972505+0000", 2023-10-09T04:36:16.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00034054199999999999 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:173012 6.c 6:3e1d0f82:::benchmark_data_smithi082_128868_object85674:head [delete] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:16.924812+0000", 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.051506415, 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.051780229999999997, 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:36:16.996 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 173012 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924812+0000", 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924812+0000", 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924810+0000", 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0219999999999993e-06 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:36:16.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924819+0000", 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3686000000000001e-05 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924832+0000", 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924839+0000", 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0403999999999997e-05 2023-10-09T04:36:16.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.924930+0000", 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000272806 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.925202+0000", 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00091604200000000005 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:17.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.926118+0000", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.017701947999999999 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.943820+0000", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6929999999999999e-05 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:16.943837+0000", 2023-10-09T04:36:17.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.019025403 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: ], 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 4 2023-10-09T04:36:17.002 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:36:17.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:17.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:17.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:17.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:17.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:17.228 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:17.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:17.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:17.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:17.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:17.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:17.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:17.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:17.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:17.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:17.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:17.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:17 smithi082 bash[77063]: cluster 2023-10-09T04:36:16.400860+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v644: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23257/10125 objects degraded (229.699%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:17.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:17 smithi082 bash[79733]: cluster 2023-10-09T04:36:16.400860+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v644: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23257/10125 objects degraded (229.699%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:17.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:17.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:17.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:17.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:18.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:17 smithi153 bash[59565]: cluster 2023-10-09T04:36:16.400860+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v644: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23257/10125 objects degraded (229.699%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:36:18.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:36:18.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:18.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:18.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:18.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:18.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.447 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:18.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:18.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:18.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:18.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:18.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:18.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:36:18.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:18.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:18.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:19.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:19.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:19.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:19.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:19.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:19.563 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:19.666 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:19.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:19.699 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 87335 objects 2023-10-09T04:36:19.700 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :128.815 2023-10-09T04:36:19.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:19.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:19.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:19.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:19 smithi153 bash[59565]: cluster 2023-10-09T04:36:18.401582+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v645: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 132 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23073/6186 objects degraded (372.987%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:19.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:19 smithi153 bash[59565]: cluster 2023-10-09T04:36:19.606086+0000 mon.a (mon.0) 979 : cluster 3 Health check update: Degraded data redundancy: 23073/6186 objects degraded (372.987%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:19.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:19.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:19.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:19.969 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T04:36:19.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T04:36:19.977 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:19] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-09T04:36:19.977 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:19 smithi082 bash[79733]: cluster 2023-10-09T04:36:18.401582+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v645: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 132 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23073/6186 objects degraded (372.987%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:19.978 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:19 smithi082 bash[79733]: cluster 2023-10-09T04:36:19.606086+0000 mon.a (mon.0) 979 : cluster 3 Health check update: Degraded data redundancy: 23073/6186 objects degraded (372.987%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:19.978 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:19 smithi082 bash[77063]: cluster 2023-10-09T04:36:18.401582+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v645: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 132 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 23073/6186 objects degraded (372.987%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:19.978 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:19 smithi082 bash[77063]: cluster 2023-10-09T04:36:19.606086+0000 mon.a (mon.0) 979 : cluster 3 Health check update: Degraded data redundancy: 23073/6186 objects degraded (372.987%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:20.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:20.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:20.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:20.224 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:20.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:20.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:20.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:20.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:20.601 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:20.632 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:20.736 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:20.765 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:20.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:20.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:20.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:21.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:21.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:21.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:21.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:21.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:21.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:21.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:21.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:21.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:21.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:21.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:21.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:21.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:21.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:21.782 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:21 smithi082 bash[77063]: cluster 2023-10-09T04:36:20.402563+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v646: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:21.783 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:21 smithi082 bash[79733]: cluster 2023-10-09T04:36:20.402563+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v646: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:21.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:21.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:21 smithi153 bash[59565]: cluster 2023-10-09T04:36:20.402563+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v646: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:21.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:21.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:21.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:22.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:22.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:22.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:22.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.193 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:22.296 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:22.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:22.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:22.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:22.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:22.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:22.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:22.854 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:22.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:22.931 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:22.957 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:22.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:22.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:23.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:23.107 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:23.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:23.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:23.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:23.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:23.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:23.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:23.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:23.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:23.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:23.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:23.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:23.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:23.753 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:36:23.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:23.876 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:23.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:23.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:23 smithi082 bash[77063]: cluster 2023-10-09T04:36:22.403303+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v647: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:23.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:23 smithi082 bash[77063]: audit 2023-10-09T04:36:23.454416+0000 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:36:23.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:23 smithi082 bash[79733]: cluster 2023-10-09T04:36:22.403303+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v647: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:23.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:23 smithi082 bash[79733]: audit 2023-10-09T04:36:23.454416+0000 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:36:23.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:23.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:23 smithi153 bash[59565]: cluster 2023-10-09T04:36:22.403303+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v647: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 31 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23073/1365 objects degraded (1690.330%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:36:23.994 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:23 smithi153 bash[59565]: audit 2023-10-09T04:36:23.454416+0000 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:36:24.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:24.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:24.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:24.249 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:24.283 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:24.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:24.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:24.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:24.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:24.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:24.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:24.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:24.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:24.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.732570+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.734427+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.735420+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.736013+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.737070+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: audit 2023-10-09T04:36:23.737315+0000 mon.a (mon.0) 981 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.737399+0000 mon.a (mon.0) 982 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.738614+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:23.742189+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.607177+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 23073/1365 objects degraded (1690.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.618081+0000 mon.a (mon.0) 984 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.732570+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.734427+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.735420+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.736013+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.918 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.737070+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: audit 2023-10-09T04:36:23.737315+0000 mon.a (mon.0) 981 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.737399+0000 mon.a (mon.0) 982 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.738614+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:23.742189+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.607177+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 23073/1365 objects degraded (1690.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:24.919 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:24 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.618081+0000 mon.a (mon.0) 984 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.732570+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.734427+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.735420+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.736013+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.737070+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: audit 2023-10-09T04:36:23.737315+0000 mon.a (mon.0) 981 : audit 1 from='client.? 172.21.15.82:0/3087293070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:36:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.737399+0000 mon.a (mon.0) 982 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:36:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.738614+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:23.742189+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T04:36:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.607177+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 23073/1365 objects degraded (1690.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:24.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:24 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.618081+0000 mon.a (mon.0) 984 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:36:24.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:24.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:25.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:25.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:25.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:25.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:25.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:25.447 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:25.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:25.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:25.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:25.764 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.404103+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v649: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.3 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 23022/120 objects degraded (19185.000%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.613378+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.404103+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v649: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.3 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 23022/120 objects degraded (19185.000%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.613378+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.615876+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.619257+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.619410+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.620670+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.621180+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.765 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.621448+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.622897+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.625518+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:24.627461+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.766 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.614590+0000 mon.a (mon.0) 985 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:36:25.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.615876+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.619257+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.619410+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.620670+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.621180+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.621448+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.622897+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.625518+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:24.627461+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:25 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.614590+0000 mon.a (mon.0) 985 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:36:25.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:25.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:25.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.938 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.404103+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v649: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.3 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 23022/120 objects degraded (19185.000%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.613378+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.615876+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.619257+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.619410+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.620670+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.621180+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.621448+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.622897+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.625518+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:24.627461+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T04:36:25.939 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:25 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.614590+0000 mon.a (mon.0) 985 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:36:25.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:25.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:26.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:26.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:26.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:26.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:26.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:36:26.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:26.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:26.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:26.499 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:36:26.522 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:36:26.549 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:36:26.576 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:36:26.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:26.603 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:36:26.621 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:26.624 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:26.631 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:36:26.658 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:36:26.681 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:36:26.683 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.684 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.684 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.684 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.684 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.685 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.685 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.685 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:36:26.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:26.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:26.803 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:26.803 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:26.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:26.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.614718+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.616205+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.617370+0000 osd.2 (osd.2) 1 : cluster 0 6.2 starting backfill to osd.4 from (0'0,0'0] MAX to 161'10932 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.640449+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: cluster 2023-10-09T04:36:25.671969+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[77063]: audit 2023-10-09T04:36:26.470672+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.82:0/1860659290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.614718+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.616205+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.617370+0000 osd.2 (osd.2) 1 : cluster 0 6.2 starting backfill to osd.4 from (0'0,0'0] MAX to 161'10932 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.640449+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: cluster 2023-10-09T04:36:25.671969+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:26 smithi082 bash[79733]: audit 2023-10-09T04:36:26.470672+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.82:0/1860659290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:26.986 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.614718+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.616205+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.617370+0000 osd.2 (osd.2) 1 : cluster 0 6.2 starting backfill to osd.4 from (0'0,0'0] MAX to 161'10932 2023-10-09T04:36:26.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.640449+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: cluster 2023-10-09T04:36:25.671969+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T04:36:26.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:26 smithi153 bash[59565]: audit 2023-10-09T04:36:26.470672+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.82:0/1860659290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:27.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:27.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:27.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:27.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:27.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:27.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:27.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:27.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:27.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:27.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:27.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:27.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:27.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:27.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:36:27.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:27.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:27.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:27.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[77063]: cluster 2023-10-09T04:36:26.404975+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v652: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 23022/84 objects degraded (27407.143%); 544 KiB/s, 78 objects/s recovering 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[77063]: cluster 2023-10-09T04:36:26.768468+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[77063]: cluster 2023-10-09T04:36:26.768939+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[77063]: cluster 2023-10-09T04:36:26.771853+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[77063]: cluster 2023-10-09T04:36:26.772827+0000 mon.a (mon.0) 987 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[79733]: cluster 2023-10-09T04:36:26.404975+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v652: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 23022/84 objects degraded (27407.143%); 544 KiB/s, 78 objects/s recovering 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[79733]: cluster 2023-10-09T04:36:26.768468+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[79733]: cluster 2023-10-09T04:36:26.768939+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[79733]: cluster 2023-10-09T04:36:26.771853+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:27.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:27 smithi082 bash[79733]: cluster 2023-10-09T04:36:26.772827+0000 mon.a (mon.0) 987 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:36:27.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:27.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:28.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:28.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:28.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:28.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:27 smithi153 bash[59565]: cluster 2023-10-09T04:36:26.404975+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v652: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 23022/84 objects degraded (27407.143%); 544 KiB/s, 78 objects/s recovering 2023-10-09T04:36:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:27 smithi153 bash[59565]: cluster 2023-10-09T04:36:26.768468+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:27 smithi153 bash[59565]: cluster 2023-10-09T04:36:26.768939+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:27 smithi153 bash[59565]: cluster 2023-10-09T04:36:26.771853+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T04:36:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:27 smithi153 bash[59565]: cluster 2023-10-09T04:36:26.772827+0000 mon.a (mon.0) 987 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:36:28.265 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:28.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:28.394 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:28.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:28.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:28.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:28.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:28.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:28.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:28.813 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:28.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:28.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:28.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:28.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:29.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:28 smithi153 bash[59565]: cluster 2023-10-09T04:36:27.770796+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:28 smithi153 bash[59565]: cluster 2023-10-09T04:36:27.773902+0000 mon.a (mon.0) 988 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:36:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:28 smithi153 bash[59565]: cluster 2023-10-09T04:36:27.774426+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:28 smithi153 bash[59565]: cluster 2023-10-09T04:36:27.777691+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:29.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.189 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[79733]: cluster 2023-10-09T04:36:27.770796+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.189 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[79733]: cluster 2023-10-09T04:36:27.773902+0000 mon.a (mon.0) 988 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:36:29.189 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[79733]: cluster 2023-10-09T04:36:27.774426+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.189 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[79733]: cluster 2023-10-09T04:36:27.777691+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[77063]: cluster 2023-10-09T04:36:27.770796+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[77063]: cluster 2023-10-09T04:36:27.773902+0000 mon.a (mon.0) 988 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:36:29.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[77063]: cluster 2023-10-09T04:36:27.774426+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:28 smithi082 bash[77063]: cluster 2023-10-09T04:36:27.777691+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:29.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:29.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:29.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:29.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:29.687 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:29.718 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:29.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:29.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:29.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:29.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:29] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: cluster 2023-10-09T04:36:28.274246+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: cluster 2023-10-09T04:36:28.405639+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v655: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 22751/84 objects degraded (27084.524%); 0 B/s, 330 objects/s recovering 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: cluster 2023-10-09T04:36:28.769691+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: cluster 2023-10-09T04:36:28.772276+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: audit 2023-10-09T04:36:29.269378+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:29.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[77063]: cluster 2023-10-09T04:36:29.609445+0000 mon.a (mon.0) 990 : cluster 3 Health check update: Degraded data redundancy: 22751/84 objects degraded (27084.524%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: cluster 2023-10-09T04:36:28.274246+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: cluster 2023-10-09T04:36:28.405639+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v655: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 22751/84 objects degraded (27084.524%); 0 B/s, 330 objects/s recovering 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: cluster 2023-10-09T04:36:28.769691+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: cluster 2023-10-09T04:36:28.772276+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: audit 2023-10-09T04:36:29.269378+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:29.967 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:29 smithi082 bash[79733]: cluster 2023-10-09T04:36:29.609445+0000 mon.a (mon.0) 990 : cluster 3 Health check update: Degraded data redundancy: 22751/84 objects degraded (27084.524%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:29.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:29.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:30.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: cluster 2023-10-09T04:36:28.274246+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: cluster 2023-10-09T04:36:28.405639+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v655: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 22751/84 objects degraded (27084.524%); 0 B/s, 330 objects/s recovering 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: cluster 2023-10-09T04:36:28.769691+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: cluster 2023-10-09T04:36:28.772276+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: audit 2023-10-09T04:36:29.269378+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:30.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:29 smithi153 bash[59565]: cluster 2023-10-09T04:36:29.609445+0000 mon.a (mon.0) 990 : cluster 3 Health check update: Degraded data redundancy: 22751/84 objects degraded (27084.524%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:30.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:30.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:30.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:30.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:30.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:30.618 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:30.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:30.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:30.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:30.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:30.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:30.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:31.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:31.022 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:30 smithi082 bash[77063]: audit 2023-10-09T04:36:30.545813+0000 mon.a (mon.0) 991 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:31.022 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:30 smithi082 bash[79733]: audit 2023-10-09T04:36:30.545813+0000 mon.a (mon.0) 991 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:31.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:31.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:31.151 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:30 smithi153 bash[59565]: audit 2023-10-09T04:36:30.545813+0000 mon.a (mon.0) 991 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:36:31.168 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:31.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:31.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:31.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:31.442 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:31.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:31.578 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:31.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:31.711 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:31.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:31.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:31.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:31.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:31.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[77063]: cluster 2023-10-09T04:36:30.406757+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v656: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 228 objects/s recovering 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[77063]: cluster 2023-10-09T04:36:30.795056+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[77063]: cluster 2023-10-09T04:36:30.795857+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[77063]: cluster 2023-10-09T04:36:30.798028+0000 mon.a (mon.0) 992 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[77063]: cluster 2023-10-09T04:36:30.799972+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[79733]: cluster 2023-10-09T04:36:30.406757+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v656: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 228 objects/s recovering 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[79733]: cluster 2023-10-09T04:36:30.795056+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[79733]: cluster 2023-10-09T04:36:30.795857+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[79733]: cluster 2023-10-09T04:36:30.798028+0000 mon.a (mon.0) 992 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:36:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:31 smithi082 bash[79733]: cluster 2023-10-09T04:36:30.799972+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:32.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:32.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:32.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:32.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:31 smithi153 bash[59565]: cluster 2023-10-09T04:36:30.406757+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v656: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 228 objects/s recovering 2023-10-09T04:36:32.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:31 smithi153 bash[59565]: cluster 2023-10-09T04:36:30.795056+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:32.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:31 smithi153 bash[59565]: cluster 2023-10-09T04:36:30.795857+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:32.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:31 smithi153 bash[59565]: cluster 2023-10-09T04:36:30.798028+0000 mon.a (mon.0) 992 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:36:32.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:31 smithi153 bash[59565]: cluster 2023-10-09T04:36:30.799972+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T04:36:32.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:32.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:32.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:36:32.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:32.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:32.463 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:32.490 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:32.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:32.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:32.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:32.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:32.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:32.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:32.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[79733]: cluster 2023-10-09T04:36:31.803628+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[79733]: cluster 2023-10-09T04:36:31.807035+0000 mon.a (mon.0) 993 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:36:32.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[79733]: cluster 2023-10-09T04:36:31.842186+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[79733]: cluster 2023-10-09T04:36:31.842271+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:32.899 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[77063]: cluster 2023-10-09T04:36:31.803628+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.899 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[77063]: cluster 2023-10-09T04:36:31.807035+0000 mon.a (mon.0) 993 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:36:32.899 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[77063]: cluster 2023-10-09T04:36:31.842186+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.899 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:32 smithi082 bash[77063]: cluster 2023-10-09T04:36:31.842271+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:32.912 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:32.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:32.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:33.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:33.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:33.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:33.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:33.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:32 smithi153 bash[59565]: cluster 2023-10-09T04:36:31.803628+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:33.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:32 smithi153 bash[59565]: cluster 2023-10-09T04:36:31.807035+0000 mon.a (mon.0) 993 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:36:33.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:32 smithi153 bash[59565]: cluster 2023-10-09T04:36:31.842186+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:33.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:32 smithi153 bash[59565]: cluster 2023-10-09T04:36:31.842271+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T04:36:33.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:33.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:33.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:33.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:33.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:33.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:33.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.680 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:33.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:33.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:33.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:33.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:34.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:34.068 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:33 smithi153 bash[59565]: cluster 2023-10-09T04:36:32.407386+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v659: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 234 objects/s recovering 2023-10-09T04:36:34.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:34.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:34.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:34.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:33 smithi082 bash[79733]: cluster 2023-10-09T04:36:32.407386+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v659: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 234 objects/s recovering 2023-10-09T04:36:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:33 smithi082 bash[77063]: cluster 2023-10-09T04:36:32.407386+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v659: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 22751/84 objects degraded (27084.524%); 0 B/s, 234 objects/s recovering 2023-10-09T04:36:34.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:34.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:34.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:34.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:34.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:34.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:34.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:34.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:34.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:34.810 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:34.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:34.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:34.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:34.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:35.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:35.084 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:35.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:35.187 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:35.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:35.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:35.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:35.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:35.475 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:35.578 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:35.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:35.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:35.731 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:35.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:35 smithi082 bash[77063]: cluster 2023-10-09T04:36:34.408069+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v660: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 21306/84 objects degraded (25364.286%); 0 B/s, 272 objects/s recovering 2023-10-09T04:36:35.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:35 smithi082 bash[77063]: cluster 2023-10-09T04:36:34.810252+0000 mon.a (mon.0) 994 : cluster 3 Health check update: Degraded data redundancy: 21306/84 objects degraded (25364.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:35.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:35 smithi082 bash[79733]: cluster 2023-10-09T04:36:34.408069+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v660: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 21306/84 objects degraded (25364.286%); 0 B/s, 272 objects/s recovering 2023-10-09T04:36:35.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:35 smithi082 bash[79733]: cluster 2023-10-09T04:36:34.810252+0000 mon.a (mon.0) 994 : cluster 3 Health check update: Degraded data redundancy: 21306/84 objects degraded (25364.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:35.759 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:35.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:35.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:35.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:35.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:35 smithi153 bash[59565]: cluster 2023-10-09T04:36:34.408069+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v660: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 21306/84 objects degraded (25364.286%); 0 B/s, 272 objects/s recovering 2023-10-09T04:36:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:35 smithi153 bash[59565]: cluster 2023-10-09T04:36:34.810252+0000 mon.a (mon.0) 994 : cluster 3 Health check update: Degraded data redundancy: 21306/84 objects degraded (25364.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:36.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:36.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:36.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:36.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:36.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:36.183 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:36.183 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:36.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:36.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:36.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:36.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:36.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:36.539 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:36.567 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:36.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:36.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:36.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:36.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:36.872 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:36.924 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:35.787370+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:35.797442+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:36.141853+0000 mon.a (mon.0) 997 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:36.148176+0000 mon.a (mon.0) 998 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:36.620018+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:36.621310+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:36.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:36 smithi153 bash[59565]: audit 2023-10-09T04:36:36.626689+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:36.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:35.787370+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:35.797442+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:36.141853+0000 mon.a (mon.0) 997 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:36.148176+0000 mon.a (mon.0) 998 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:36.620018+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:36.621310+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[77063]: audit 2023-10-09T04:36:36.626689+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:35.787370+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:35.797442+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:36.141853+0000 mon.a (mon.0) 997 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:36.148176+0000 mon.a (mon.0) 998 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:36.620018+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:36.621310+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:36.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:36 smithi082 bash[79733]: audit 2023-10-09T04:36:36.626689+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:37.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:37.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:37.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:36:37.186 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:37.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:37.325 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:37.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:37.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:37.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:37.595 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:37.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:37.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:37.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:37.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:37.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:37.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:37.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:38.001 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:37 smithi082 bash[77063]: cluster 2023-10-09T04:36:36.408678+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v661: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:38.002 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:37 smithi082 bash[79733]: cluster 2023-10-09T04:36:36.408678+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v661: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:38.020 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:38.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:37 smithi153 bash[59565]: cluster 2023-10-09T04:36:36.408678+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v661: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:38.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:38.255 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:38.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:38.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:38.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:38.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:38.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:38.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:38.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:38.917 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:38.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:38.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:39.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:39.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:39.169 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:39.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:39.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:39.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:39.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:39.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:39.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:39.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:39.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:39.606 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:39.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:39.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:39.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:39.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:39.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:39.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:39.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:39.993 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:39 smithi082 bash[77063]: cluster 2023-10-09T04:36:38.409431+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v662: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:39.994 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:39] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-09T04:36:39.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:39 smithi082 bash[79733]: cluster 2023-10-09T04:36:38.409431+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v662: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:39.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:40.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:40.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:40.115 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:39 smithi153 bash[59565]: cluster 2023-10-09T04:36:38.409431+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v662: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:40.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:40.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:40.277 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:40.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:40.410 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:40.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:40.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:40.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:40.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:40.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:40.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:40.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:40.808 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:36:40.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:40.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:40 smithi082 bash[79733]: audit 2023-10-09T04:36:40.717078+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.82:0/1232063312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:40.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:40 smithi082 bash[79733]: audit 2023-10-09T04:36:40.717458+0000 mon.a (mon.0) 1002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:40.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:40 smithi082 bash[77063]: audit 2023-10-09T04:36:40.717078+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.82:0/1232063312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:40.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:40 smithi082 bash[77063]: audit 2023-10-09T04:36:40.717458+0000 mon.a (mon.0) 1002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:40.918 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:40.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:41.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:41.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:41.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:41.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:41.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:40 smithi153 bash[59565]: audit 2023-10-09T04:36:40.717078+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.82:0/1232063312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:41.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:40 smithi153 bash[59565]: audit 2023-10-09T04:36:40.717458+0000 mon.a (mon.0) 1002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:41.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:41.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:41.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:41.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:41.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:41.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:41.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:41.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:41.928 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:41.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:41.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:42.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:42.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: cluster 2023-10-09T04:36:40.410019+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v663: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 189 objects/s recovering 2023-10-09T04:36:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: cluster 2023-10-09T04:36:40.791120+0000 mon.a (mon.0) 1003 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: cluster 2023-10-09T04:36:40.799601+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: cluster 2023-10-09T04:36:40.799984+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: audit 2023-10-09T04:36:40.802823+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:42.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:41 smithi153 bash[59565]: cluster 2023-10-09T04:36:40.802943+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:36:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:42.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:42.171 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: cluster 2023-10-09T04:36:40.410019+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v663: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 189 objects/s recovering 2023-10-09T04:36:42.171 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: cluster 2023-10-09T04:36:40.791120+0000 mon.a (mon.0) 1003 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: cluster 2023-10-09T04:36:40.799601+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: cluster 2023-10-09T04:36:40.799984+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: audit 2023-10-09T04:36:40.802823+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[77063]: cluster 2023-10-09T04:36:40.802943+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: cluster 2023-10-09T04:36:40.410019+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v663: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 189 objects/s recovering 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: cluster 2023-10-09T04:36:40.791120+0000 mon.a (mon.0) 1003 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: cluster 2023-10-09T04:36:40.799601+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: cluster 2023-10-09T04:36:40.799984+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T04:36:42.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: audit 2023-10-09T04:36:40.802823+0000 mon.a (mon.0) 1004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:42.173 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:41 smithi082 bash[79733]: cluster 2023-10-09T04:36:40.802943+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:36:42.186 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:42.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:42.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:42.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:42.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:42.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:42.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:42.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:42.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:42.701 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:42.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:42.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:42.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:42.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:42.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:43.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:43.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:43.195 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:43.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.356 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:43.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:43.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:43.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:43.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:43.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:43.735 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:43.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:43.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:43.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:43 smithi082 bash[77063]: cluster 2023-10-09T04:36:42.410481+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v665: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 182 objects/s recovering 2023-10-09T04:36:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:43 smithi082 bash[79733]: cluster 2023-10-09T04:36:42.410481+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v665: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 182 objects/s recovering 2023-10-09T04:36:43.996 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:43.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:44.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:44.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:43 smithi153 bash[59565]: cluster 2023-10-09T04:36:42.410481+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v665: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20447/84 objects degraded (24341.667%); 0 B/s, 182 objects/s recovering 2023-10-09T04:36:44.124 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:44.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:44.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:44.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:44.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:44.387 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:44.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:44.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:44.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:44.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:44.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:44.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:44.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:44.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:44.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:44.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:45.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:45.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.151 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:45.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:45.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:45.393 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[79733]: audit 2023-10-09T04:36:44.277778+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:45.393 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[79733]: audit 2023-10-09T04:36:44.279300+0000 mon.a (mon.0) 1007 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:45.393 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[79733]: cluster 2023-10-09T04:36:44.411015+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v666: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 203 objects/s recovering 2023-10-09T04:36:45.394 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[79733]: cluster 2023-10-09T04:36:44.615049+0000 mon.a (mon.0) 1008 : cluster 3 Health check update: Degraded data redundancy: 20447/84 objects degraded (24341.667%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.394 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[77063]: audit 2023-10-09T04:36:44.277778+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:45.394 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[77063]: audit 2023-10-09T04:36:44.279300+0000 mon.a (mon.0) 1007 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:45.394 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[77063]: cluster 2023-10-09T04:36:44.411015+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v666: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 203 objects/s recovering 2023-10-09T04:36:45.394 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:45 smithi082 bash[77063]: cluster 2023-10-09T04:36:44.615049+0000 mon.a (mon.0) 1008 : cluster 3 Health check update: Degraded data redundancy: 20447/84 objects degraded (24341.667%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:45.509 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:45.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:45.533 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:45 smithi153 bash[59565]: audit 2023-10-09T04:36:44.277778+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:45.533 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:45 smithi153 bash[59565]: audit 2023-10-09T04:36:44.279300+0000 mon.a (mon.0) 1007 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:45.533 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:45 smithi153 bash[59565]: cluster 2023-10-09T04:36:44.411015+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v666: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 203 objects/s recovering 2023-10-09T04:36:45.533 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:45 smithi153 bash[59565]: cluster 2023-10-09T04:36:44.615049+0000 mon.a (mon.0) 1008 : cluster 3 Health check update: Degraded data redundancy: 20447/84 objects degraded (24341.667%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:45.628 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:45.629 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:45.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:45.664 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:45.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:45.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:45.903 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:45.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:46.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:46.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:46.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:46.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:46.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:46.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:46.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:46.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:46.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:46.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:36:46.542 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:46.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:46.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:46.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:46.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:46.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:46.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:46.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:47.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:47.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:47.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:47.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:47.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:47.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:47.490 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:47.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:47.738 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:47 smithi082 bash[79733]: cluster 2023-10-09T04:36:46.411627+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v667: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:47.738 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:47 smithi082 bash[77063]: cluster 2023-10-09T04:36:46.411627+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v667: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:47.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:47.750 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:47 smithi153 bash[59565]: cluster 2023-10-09T04:36:46.411627+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v667: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:47.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:47.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:47.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:47.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:48.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:48.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:48.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:48.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:48.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:48.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:48.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.422 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:48.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:48.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.675 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:48.780 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:48.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:48.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:48.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:48.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:49.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:49.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:49.186 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:49.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:49.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:49.345 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:49.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:49.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:49.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:49.591 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:49 smithi082 bash[79733]: cluster 2023-10-09T04:36:48.412477+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v668: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:49.591 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:49 smithi082 bash[77063]: cluster 2023-10-09T04:36:48.412477+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v668: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:49.606 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:49.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:49.722 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:49 smithi153 bash[59565]: cluster 2023-10-09T04:36:48.412477+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v668: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20230/84 objects degraded (24083.333%); 0 B/s, 117 objects/s recovering 2023-10-09T04:36:49.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:49.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:49.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:49.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:49.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:49.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:49] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-09T04:36:50.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:50.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:50.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:50.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:50.238 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:50.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:50.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:50.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:50.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:36:50.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:50.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:50 smithi153 bash[59565]: cluster 2023-10-09T04:36:49.616551+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 20230/84 objects degraded (24083.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:50.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:50 smithi082 bash[79733]: cluster 2023-10-09T04:36:49.616551+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 20230/84 objects degraded (24083.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:50 smithi082 bash[77063]: cluster 2023-10-09T04:36:49.616551+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 20230/84 objects degraded (24083.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:50.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:50.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:50.913 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:50.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:51.016 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:51.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:51.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:51.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:51.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:51.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:51.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:51.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:51.558 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:51.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:51.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:51 smithi082 bash[77063]: cluster 2023-10-09T04:36:50.413600+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v669: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 142 objects/s recovering 2023-10-09T04:36:51.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:51 smithi082 bash[79733]: cluster 2023-10-09T04:36:50.413600+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v669: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 142 objects/s recovering 2023-10-09T04:36:51.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:51.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:51.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:51 smithi153 bash[59565]: cluster 2023-10-09T04:36:50.413600+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v669: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 142 objects/s recovering 2023-10-09T04:36:51.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:51.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:52.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:52.080 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:52.183 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:52.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:52.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:52.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:52.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:52.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:52.578 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:52.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:52.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:52.706 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:52.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:52.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:52.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:52.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:52.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:53.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:53.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:53.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:53.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:53.262 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:53.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:53.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:53.517 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:53.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:53.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:53.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:53.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.886 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:53 smithi082 bash[77063]: cluster 2023-10-09T04:36:52.414073+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v670: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 122 objects/s recovering 2023-10-09T04:36:53.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:53 smithi082 bash[79733]: cluster 2023-10-09T04:36:52.414073+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v670: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 122 objects/s recovering 2023-10-09T04:36:53.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:53.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:53.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:53 smithi153 bash[59565]: cluster 2023-10-09T04:36:52.414073+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v670: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19503/84 objects degraded (23217.857%); 0 B/s, 122 objects/s recovering 2023-10-09T04:36:53.978 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:36:54.000 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:36:54.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:54.022 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:36:54.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:54.049 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:36:54.075 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:36:54.101 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:36:54.125 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:36:54.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:54.148 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.151 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.152 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.152 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:36:54.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:54.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:54.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:54.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:54.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:54.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:54.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:54.647 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_1' removed 2023-10-09T04:36:54.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: audit 2023-10-09T04:36:53.631500+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.82:0/3959975817' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: audit 2023-10-09T04:36:53.631822+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.649 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: audit 2023-10-09T04:36:53.951826+0000 mon.a (mon.0) 1011 : audit 0 from='client.? 172.21.15.82:0/2707414808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:54.649 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: audit 2023-10-09T04:36:54.209631+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.82:0/2436410491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.649 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: audit 2023-10-09T04:36:54.210213+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.649 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[77063]: cluster 2023-10-09T04:36:54.617978+0000 mon.a (mon.0) 1013 : cluster 3 Health check update: Degraded data redundancy: 19503/84 objects degraded (23217.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:54.649 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:36:54.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:54.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: audit 2023-10-09T04:36:53.631500+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.82:0/3959975817' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: audit 2023-10-09T04:36:53.631822+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: audit 2023-10-09T04:36:53.951826+0000 mon.a (mon.0) 1011 : audit 0 from='client.? 172.21.15.82:0/2707414808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: audit 2023-10-09T04:36:54.209631+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.82:0/2436410491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: audit 2023-10-09T04:36:54.210213+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:54 smithi153 bash[59565]: cluster 2023-10-09T04:36:54.617978+0000 mon.a (mon.0) 1013 : cluster 3 Health check update: Degraded data redundancy: 19503/84 objects degraded (23217.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:54.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:54.820 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:36:54.820 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:36:54.820 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:36:54.820 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:36:54.820 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:36:54.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:54.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:54.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:54.971 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:54.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:54.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: audit 2023-10-09T04:36:53.631500+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.82:0/3959975817' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: audit 2023-10-09T04:36:53.631822+0000 mon.a (mon.0) 1010 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:36:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: audit 2023-10-09T04:36:53.951826+0000 mon.a (mon.0) 1011 : audit 0 from='client.? 172.21.15.82:0/2707414808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: audit 2023-10-09T04:36:54.209631+0000 mon.c (mon.2) 74 : audit 1 from='client.? 172.21.15.82:0/2436410491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: audit 2023-10-09T04:36:54.210213+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:36:54.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:54 smithi082 bash[79733]: cluster 2023-10-09T04:36:54.617978+0000 mon.a (mon.0) 1013 : cluster 3 Health check update: Degraded data redundancy: 19503/84 objects degraded (23217.857%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:55.074 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:55.103 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:55.206 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:55.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:55.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:55.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:55.466 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:55.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:55.597 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:55.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:36:55.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:55.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:55.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:55.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:55.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:55 smithi153 bash[59565]: cluster 2023-10-09T04:36:54.414693+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v671: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18905/84 objects degraded (22505.952%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:55.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:55 smithi153 bash[59565]: cluster 2023-10-09T04:36:54.621651+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:55.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:55 smithi153 bash[59565]: audit 2023-10-09T04:36:54.624233+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:36:55.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:55 smithi153 bash[59565]: audit 2023-10-09T04:36:54.624807+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:36:55.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:55 smithi153 bash[59565]: cluster 2023-10-09T04:36:54.624881+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:36:55.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:55.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[79733]: cluster 2023-10-09T04:36:54.414693+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v671: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18905/84 objects degraded (22505.952%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[79733]: cluster 2023-10-09T04:36:54.621651+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[79733]: audit 2023-10-09T04:36:54.624233+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[79733]: audit 2023-10-09T04:36:54.624807+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[79733]: cluster 2023-10-09T04:36:54.624881+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[77063]: cluster 2023-10-09T04:36:54.414693+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v671: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18905/84 objects degraded (22505.952%); 0 B/s, 168 objects/s recovering 2023-10-09T04:36:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[77063]: cluster 2023-10-09T04:36:54.621651+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[77063]: audit 2023-10-09T04:36:54.624233+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:36:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[77063]: audit 2023-10-09T04:36:54.624807+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:36:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:55 smithi082 bash[77063]: cluster 2023-10-09T04:36:54.624881+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:36:55.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:56.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:56.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:56.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:56.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.411 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:56.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:56.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:56.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:56.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:56.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:56 smithi153 bash[59565]: cluster 2023-10-09T04:36:55.682106+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:56.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:56 smithi082 bash[77063]: cluster 2023-10-09T04:36:55.682106+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:56.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:56 smithi082 bash[79733]: cluster 2023-10-09T04:36:55.682106+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T04:36:56.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:56.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:56.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:57.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:57.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:57.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:57.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:57.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:57.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:57.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:36:57.439 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:36:57.440 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:36:57.440 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T04:36:57.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T04:36:57.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:57.578 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:57.718 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.730 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:57 smithi153 bash[59565]: cluster 2023-10-09T04:36:56.415503+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v673: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:57.730 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:57 smithi153 bash[59565]: cluster 2023-10-09T04:36:56.647385+0000 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18905/84 objects degraded (22505.952%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T04:36:57.750 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:57.854 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:57.866 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:57 smithi082 bash[77063]: cluster 2023-10-09T04:36:56.415503+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v673: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:57.866 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:57 smithi082 bash[77063]: cluster 2023-10-09T04:36:56.647385+0000 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18905/84 objects degraded (22505.952%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T04:36:57.866 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:57 smithi082 bash[79733]: cluster 2023-10-09T04:36:56.415503+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v673: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:57.866 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:57 smithi082 bash[79733]: cluster 2023-10-09T04:36:56.647385+0000 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18905/84 objects degraded (22505.952%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T04:36:57.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:57.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:57.921 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:57.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:58.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:58.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:58.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:36:58.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:58.277 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:58.381 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:58.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:58.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:58.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:58.645 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:58.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:58.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.809 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:58.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:58.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:59.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:59.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:59.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:59.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:59.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:59.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:59.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:36:59.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:59.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:59.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:59.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:59.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:59.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:36:59.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:59 smithi153 bash[59565]: cluster 2023-10-09T04:36:58.416385+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v674: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:59 smithi153 bash[59565]: audit 2023-10-09T04:36:59.276636+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:36:59 smithi153 bash[59565]: audit 2023-10-09T04:36:59.278410+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:59.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:36:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[77063]: cluster 2023-10-09T04:36:58.416385+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v674: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[77063]: audit 2023-10-09T04:36:59.276636+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:59.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[77063]: audit 2023-10-09T04:36:59.278410+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:59.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:36:59] "GET /metrics HTTP/1.1" 200 34103 "" "Prometheus/2.33.4" 2023-10-09T04:36:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[79733]: cluster 2023-10-09T04:36:58.416385+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v674: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:36:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[79733]: audit 2023-10-09T04:36:59.276636+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:36:59.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:36:59 smithi082 bash[79733]: audit 2023-10-09T04:36:59.278410+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:59.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:00.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:00.134 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:00.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:00.238 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:00.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:00.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:00.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:00.420 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:00.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:00.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:00.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:00.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:00.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:00.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:00.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:00.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:00 smithi082 bash[79733]: audit 2023-10-09T04:36:59.701333+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:00.812 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:00 smithi082 bash[77063]: audit 2023-10-09T04:36:59.701333+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:00.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:00.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:00.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:01.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:01.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:01.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:00 smithi153 bash[59565]: audit 2023-10-09T04:36:59.701333+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:01.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:01.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:01.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:01.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:01.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:01.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:01.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:01.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:01.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:01.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:01.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:01 smithi082 bash[77063]: cluster 2023-10-09T04:37:00.417331+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v675: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:01.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:01 smithi082 bash[79733]: cluster 2023-10-09T04:37:00.417331+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v675: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:01.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:01.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:01.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:01.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:01.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:01 smithi153 bash[59565]: cluster 2023-10-09T04:37:00.417331+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v675: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:01.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:02.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:02.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:02.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:02.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:02.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:02.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:02.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:02.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:02.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:02.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:02.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:02.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:02.726 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:37:02.777 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:02 smithi082 bash[77063]: audit 2023-10-09T04:37:01.916489+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.82:0/854792863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.777 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:02 smithi082 bash[77063]: audit 2023-10-09T04:37:01.917514+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.778 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:02 smithi082 bash[79733]: audit 2023-10-09T04:37:01.916489+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.82:0/854792863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.778 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:02 smithi082 bash[79733]: audit 2023-10-09T04:37:01.917514+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:02.809 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:02.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:02.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:02 smithi153 bash[59565]: audit 2023-10-09T04:37:01.916489+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.82:0/854792863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:02 smithi153 bash[59565]: audit 2023-10-09T04:37:01.917514+0000 mon.a (mon.0) 1021 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:02.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:02.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:03.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:03.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:03.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:03.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:03.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:03.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:03.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:03.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:03.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:03.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:03.710 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:03.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:03.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:03.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:03.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.920 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110173 6.d 6.5b8345ad (undecoded) ondisk+retry+write+known_if_redirected e150)", 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.671637+0000", 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.24222325, 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.97361053900000005, 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110173 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.921 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671637+0000", 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671637+0000", 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671635+0000", 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9190000000000005e-06 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671644+0000", 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3627e-05 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671657+0000", 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6937000000000001e-05 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671674+0000", 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0071797600000000003 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.678854+0000", 2023-10-09T04:37:03.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.126e-06 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.678861+0000", 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96637379300000004 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645235+0000", 2023-10-09T04:37:03.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2320000000000001e-05 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645247+0000", 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.97361053900000005 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.926 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110174 6.d 6.a4ec68ad (undecoded) ondisk+retry+write+known_if_redirected e150)", 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.671686+0000", 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.24217360200001, 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.97359214699999996, 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110174 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671686+0000", 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671686+0000", 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671686+0000", 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.6040000000000001e-06 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671689+0000", 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3680000000000001e-06 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671695+0000", 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0249999999999996e-06 2023-10-09T04:37:03.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671699+0000", 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0071896449999999997 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.678888+0000", 2023-10-09T04:37:03.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4049999999999999e-06 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.678894+0000", 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96637717999999995 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645271+0000", 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5440000000000001e-06 2023-10-09T04:37:03.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645279+0000", 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.97359214699999996 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110183 6.a 6.4b9ef15a (undecoded) ondisk+write+known_if_redirected e150)", 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.671754+0000", 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.242106208, 2023-10-09T04:37:03.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96485722399999996, 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110183 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671754+0000", 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671754+0000", 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671753+0000", 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5795e-05 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671769+0000", 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3361e-05 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671782+0000", 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8140000000000001e-06 2023-10-09T04:37:03.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671790+0000", 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0091168159999999998 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.680907+0000", 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2720000000000003e-06 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.680913+0000", 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.95568396700000002 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.636597+0000", 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4263999999999999e-05 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.636611+0000", 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96485722399999996 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110192 6.a 6.21012b4a (undecoded) ondisk+write+known_if_redirected e150)", 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.671804+0000", 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.24205617499999, 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96484256999999995, 2023-10-09T04:37:03.938 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110192 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671804+0000", 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671804+0000", 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671803+0000", 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2939999999999999e-06 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671807+0000", 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0400000000000004e-06 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671814+0000", 2023-10-09T04:37:03.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5530000000000004e-06 2023-10-09T04:37:03.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.671822+0000", 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0091117869999999997 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.680934+0000", 2023-10-09T04:37:03.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1079999999999998e-06 2023-10-09T04:37:03.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.680940+0000", 2023-10-09T04:37:03.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.95569807900000003 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.636638+0000", 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4610000000000008e-06 2023-10-09T04:37:03.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.636646+0000", 2023-10-09T04:37:03.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96484256999999995 2023-10-09T04:37:03.946 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110202 6.7 6.7e374707 (undecoded) ondisk+write+known_if_redirected e150)", 2023-10-09T04:37:03.947 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.677327+0000", 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.23653296800001, 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96495583399999996, 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110202 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677327+0000", 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677327+0000", 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677325+0000", 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9500000000000001e-06 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677333+0000", 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0693000000000001e-05 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677344+0000", 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7580000000000003e-06 2023-10-09T04:37:03.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677351+0000", 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014156800000000001 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677492+0000", 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8439999999999998e-06 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.677500+0000", 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96477177000000003 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.642272+0000", 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1112e-05 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.642283+0000", 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96495583399999996 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110210 6.7 6.73b5a967 (undecoded) ondisk+write+known_if_redirected e150)", 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.696284+0000", 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.21757600500001, 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94605831399999996, 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.953 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110210 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696284+0000", 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696284+0000", 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696282+0000", 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1396000000000001e-05 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696294+0000", 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8056000000000001e-05 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696312+0000", 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.841e-06 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696319+0000", 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8897999999999998e-05 2023-10-09T04:37:03.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696378+0000", 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1689999999999998e-06 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.696388+0000", 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94594564999999997 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.642333+0000", 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1379999999999996e-06 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.642342+0000", 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94605831399999996 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:110211 6.d 6.8bc4f2dd (undecoded) ondisk+write+known_if_redirected e150)", 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:35.701822+0000", 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 148.21203803700001, 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94348370999999998, 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 110211 2023-10-09T04:37:03.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701822+0000", 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701822+0000", 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701820+0000", 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3549999999999994e-06 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701830+0000", 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4168000000000001e-05 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701844+0000", 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2316e-05 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701856+0000", 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2262999999999997e-05 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701919+0000", 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2269999999999993e-06 2023-10-09T04:37:03.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:35.701927+0000", 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94337203400000003 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645299+0000", 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8859999999999998e-06 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:36.645306+0000", 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.94348370999999998 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:110178 6.4 e152/151 6:2de64a70:::benchmark_data_smithi082_128868_object22840:head v 152'6910, mlcod=0'0)", 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:37.988885+0000", 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 145.92497527500001, 2023-10-09T04:37:03.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11075652, 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988885+0000", 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988885+0000", 2023-10-09T04:37:03.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988884+0000", 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2069999999999998e-06 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988892+0000", 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4943000000000001e-05 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988906+0000", 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2709999999999998e-06 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.988916+0000", 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091940015 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.080856+0000", 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.6449999999999999e-05 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.080882+0000", 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.018707259 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.099589+0000", 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1808000000000003e-05 2023-10-09T04:37:03.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.099641+0000", 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11075652 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:110195 6.4 e152/151 6:2e21989a:::benchmark_data_smithi082_128868_object22857:head v 152'6911, mlcod=0'0)", 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:37.994562+0000", 2023-10-09T04:37:03.969 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 145.91929808500001, 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10511316599999999, 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994562+0000", 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994562+0000", 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994560+0000", 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6881999999999999e-05 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994577+0000", 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5115e-05 2023-10-09T04:37:03.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994592+0000", 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2799999999999992e-06 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:37.994602+0000", 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.092749643000000007 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.087351+0000", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5975000000000001e-05 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.087377+0000", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.012270453000000001 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.099648+0000", 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.7481e-05 2023-10-09T04:37:03.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:38.099675+0000", 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10511316599999999 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:148680 6.d e157/151 6:be12bc06:::benchmark_data_smithi082_128868_object61342:head v 157'9213, mlcod=136'1387)", 2023-10-09T04:37:03.974 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:37.070404+0000", 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 86.843456203000002, 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090581811999999998, 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070404+0000", 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070404+0000", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070403+0000", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6330000000000008e-06 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070412+0000", 2023-10-09T04:37:03.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3541e-05 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070426+0000", 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.426e-06 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.070432+0000", 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0071094289999999996 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.077542+0000", 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3617000000000001e-05 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.077565+0000", 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083358305999999993 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.160924+0000", 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1830999999999998e-05 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.160986+0000", 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090581811999999998 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.979 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:148683 6.d e157/151 6:b1b2108f:::benchmark_data_smithi082_128868_object61345:head v 157'9214, mlcod=136'1387)", 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:37.081193+0000", 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 86.832666539000002, 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.098807443999999994, 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081193+0000", 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081193+0000", 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081192+0000", 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3489000000000001e-05 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081206+0000", 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5435999999999998e-05 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081231+0000", 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.072e-06 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.081239+0000", 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0084597800000000001 2023-10-09T04:37:03.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.089699+0000", 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000267796 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.089967+0000", 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.089979397000000003 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.179946+0000", 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4722000000000002e-05 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.180001+0000", 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.098807443999999994 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:148698 6.d e157/151 6:bcfe4a0c:::benchmark_data_smithi082_128868_object61360:head v 157'9215, mlcod=136'1387)", 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:37.091388+0000", 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 86.822471871000005, 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088641555999999996, 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091388+0000", 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091388+0000", 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091387+0000", 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1861e-05 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091399+0000", 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2942000000000002e-05 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091422+0000", 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5860999999999999e-05 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.091438+0000", 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.019728822999999999 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.111167+0000", 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.7993000000000001e-05 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.111195+0000", 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.068812669000000007 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.180007+0000", 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2535999999999999e-05 2023-10-09T04:37:03.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:37.180030+0000", 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088641555999999996 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.990 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:151091 6.c e157/151 6:333164ad:::benchmark_data_smithi082_128868_object63753:head v 157'9512, mlcod=136'1480)", 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:41.020160+0000", 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 82.893699884, 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090759568999999998, 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020160+0000", 2023-10-09T04:37:03.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020160+0000", 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020159+0000", 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2968e-05 2023-10-09T04:37:03.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020172+0000", 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2429000000000001e-05 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020184+0000", 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1420000000000004e-06 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.020191+0000", 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.035076997999999998 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.055268+0000", 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4452e-05 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:03.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.055323+0000", 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.055559405999999999 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.110882+0000", 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7719000000000003e-05 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.110920+0000", 2023-10-09T04:37:03.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090759568999999998 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:151103 6.c e157/151 6:3c94e6e9:::benchmark_data_smithi082_128868_object63765:head v 157'9514, mlcod=136'1480)", 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:35:41.032121+0000", 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 82.881739418999999, 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095842812999999999, 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.996 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032121+0000", 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032121+0000", 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032120+0000", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1282000000000001e-05 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032141+0000", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5681000000000001e-05 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032157+0000", 2023-10-09T04:37:03.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7590000000000003e-06 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.032165+0000", 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.054898215 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.087064+0000", 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.7644000000000002e-05 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:03.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.087091+0000", 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.040766968000000001 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.127858+0000", 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000105274 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:35:41.127963+0000", 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095842812999999999 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:168918 6.7 e160/151 6:e82b60db:::benchmark_data_smithi082_128868_object81580:head v 160'10669, mlcod=136'1445)", 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:10.265167+0000", 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 53.648692930999999, 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088783518000000006, 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.001 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265167+0000", 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265167+0000", 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:04.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265166+0000", 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3420000000000002e-06 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265174+0000", 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5224e-05 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265189+0000", 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2639999999999997e-06 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.265195+0000", 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.029578864 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.294774+0000", 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4099e-05 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.294808+0000", 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.059106760000000001 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.005 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.353915+0000", 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5738999999999998e-05 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:10.353951+0000", 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.088783518000000006 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.006 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173781 6.c e161/151 6:30af4c88:::benchmark_data_smithi082_128868_object86443:head v 161'10933, mlcod=136'1480)", 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:18.178706+0000", 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 45.735154246, 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099300316, 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178706+0000", 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178706+0000", 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178705+0000", 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1450000000000008e-06 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178713+0000", 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1905e-05 2023-10-09T04:37:04.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178725+0000", 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.678e-06 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178733+0000", 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.016062143000000001 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.194795+0000", 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1145000000000001e-05 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.194826+0000", 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083142173999999999 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.277968+0000", 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8074000000000002e-05 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278006+0000", 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099300316 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.012 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173780 6.4 e161/151 6:2f74772a:::benchmark_data_smithi082_128868_object86442:head v 161'10861, mlcod=136'1450)", 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:18.178815+0000", 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 45.735044659000003, 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099223094999999997, 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178815+0000", 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178815+0000", 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178815+0000", 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8669999999999999e-06 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178821+0000", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5049999999999996e-06 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178830+0000", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0870000000000003e-06 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.178836+0000", 2023-10-09T04:37:04.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.031361708000000002 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.210198+0000", 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9604999999999997e-05 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.210248+0000", 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.067762538999999997 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278010+0000", 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.8291999999999999e-05 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278038+0000", 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099223094999999997 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173783 6.4 e161/151 6:240fd66a:::benchmark_data_smithi082_128868_object86445:head v 161'10862, mlcod=136'1450)", 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:18.184167+0000", 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 45.729693109999999, 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.093891164999999999, 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184167+0000", 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184167+0000", 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184166+0000", 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9179999999999997e-06 2023-10-09T04:37:04.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184173+0000", 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0983e-05 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184184+0000", 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4840000000000001e-06 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.184191+0000", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.035604521 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.219795+0000", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9144000000000001e-05 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.219824+0000", 2023-10-09T04:37:04.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.058216363 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278041+0000", 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7370000000000001e-05 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278058+0000", 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.093891164999999999 2023-10-09T04:37:04.022 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173797 6.4 e161/151 6:2bea0476:::benchmark_data_smithi082_128868_object86459:head v 161'10864, mlcod=136'1450)", 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:18.211514+0000", 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 45.702345788000002, 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11182164, 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.023 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211514+0000", 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211514+0000", 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211513+0000", 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3957999999999999e-05 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211527+0000", 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5809999999999999e-05 2023-10-09T04:37:04.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211553+0000", 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0355e-05 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.211563+0000", 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.037971111000000002 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.249534+0000", 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.6304000000000001e-05 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.249560+0000", 2023-10-09T04:37:04.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.073718696 2023-10-09T04:37:04.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.323279+0000", 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6849999999999999e-05 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.323336+0000", 2023-10-09T04:37:04.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11182164 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.031 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.34995.0:173818 6.4 e161/151 6:2b5c614c:::benchmark_data_smithi082_128868_object86480:head v 161'10865, mlcod=136'1450)", 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:36:18.254263+0000", 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 45.659596602999997, 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099203498000000001, 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:04.032 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:37:04.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.033 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:37:04.033 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254263+0000", 2023-10-09T04:37:04.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:37:04.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254263+0000", 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254262+0000", 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5429999999999999e-06 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:37:04.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254272+0000", 2023-10-09T04:37:04.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4462e-05 2023-10-09T04:37:04.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254286+0000", 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0360000000000006e-06 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:37:04.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.254294+0000", 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.023791153999999998 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278086+0000", 2023-10-09T04:37:04.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2056e-05 2023-10-09T04:37:04.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:37:04.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.278108+0000", 2023-10-09T04:37:04.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.075310395000000002 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.353418+0000", 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8908e-05 2023-10-09T04:37:04.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:36:18.353467+0000", 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.099203498000000001 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.041 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.042 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.042 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:37:04.042 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:37:04.042 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:37:04.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:04.044 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: cluster 2023-10-09T04:37:02.418036+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v676: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: cluster 2023-10-09T04:37:02.705799+0000 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: cluster 2023-10-09T04:37:02.716464+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: cluster 2023-10-09T04:37:02.716882+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: audit 2023-10-09T04:37:02.719129+0000 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[77063]: cluster 2023-10-09T04:37:02.719237+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:37:04.045 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: cluster 2023-10-09T04:37:02.418036+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v676: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: cluster 2023-10-09T04:37:02.705799+0000 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: cluster 2023-10-09T04:37:02.716464+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: cluster 2023-10-09T04:37:02.716882+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: audit 2023-10-09T04:37:02.719129+0000 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:03 smithi153 bash[59565]: cluster 2023-10-09T04:37:02.719237+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:37:04.046 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: cluster 2023-10-09T04:37:02.418036+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v676: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:04.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: cluster 2023-10-09T04:37:02.705799+0000 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:04.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: cluster 2023-10-09T04:37:02.716464+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: cluster 2023-10-09T04:37:02.716882+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T04:37:04.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: audit 2023-10-09T04:37:02.719129+0000 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:04.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:03 smithi082 bash[79733]: cluster 2023-10-09T04:37:02.719237+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:37:04.047 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:04.048 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:04.048 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:04.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:04.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:04.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:04.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:04.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:04.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:04.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:04.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:04.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:04.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:04.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:04.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:04.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:04.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:37:04.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:04.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:04.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:04.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:04.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:05.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:05.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:05.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:05.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:05.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:05.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:05.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:37:05.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:05.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:05.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:05.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:05.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.914 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:05 smithi082 bash[77063]: cluster 2023-10-09T04:37:04.418564+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v678: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:05.914 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:05 smithi082 bash[79733]: cluster 2023-10-09T04:37:04.418564+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v678: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:05.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:05 smithi153 bash[59565]: cluster 2023-10-09T04:37:04.418564+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v678: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:05.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:05.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:06.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:06.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:06.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:06.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:06.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:06.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:06.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:06.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:06.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:06.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:06.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:06.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:06.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:06.854 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:06.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:06.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:06.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:07.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:07.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:07.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:07.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:07.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:07.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:07.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:07.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:07.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:07.807 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:07.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.923 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:07 smithi153 bash[59565]: cluster 2023-10-09T04:37:06.419504+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v679: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:07.939 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:07 smithi082 bash[77063]: cluster 2023-10-09T04:37:06.419504+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v679: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:07.939 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:07 smithi082 bash[79733]: cluster 2023-10-09T04:37:06.419504+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v679: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:07.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:07.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:08.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:08.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:08.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:08.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:08.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:08.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:08.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:08.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:08.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:08.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:08.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:08.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:08.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:08.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:09.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:09.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:09.148 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:09.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:09.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:09.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:09.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:09.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:09.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:09.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:09.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:09.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:09.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:09.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:09.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:09 smithi153 bash[59565]: cluster 2023-10-09T04:37:08.420041+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v680: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:09.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:09.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:10.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:10.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:10.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:10.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:10.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:10.326 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:09] "GET /metrics HTTP/1.1" 200 34103 "" "Prometheus/2.33.4" 2023-10-09T04:37:10.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:10.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:10.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:10.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:10.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:10.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:10.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:10 smithi082 bash[77063]: cluster 2023-10-09T04:37:08.420041+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v680: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:10 smithi082 bash[79733]: cluster 2023-10-09T04:37:08.420041+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v680: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:10.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:10.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:10.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:10.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:10.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:11.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:11.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:11.148 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:11.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:11.279 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:11.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:11.416 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:11.520 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:11.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:11.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:11.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:11.741 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:37:11.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:11.796 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:11 smithi082 bash[77063]: cluster 2023-10-09T04:37:10.420808+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:11.796 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:11 smithi082 bash[77063]: audit 2023-10-09T04:37:11.701056+0000 mon.a (mon.0) 1025 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:11.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:11.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:11 smithi153 bash[59565]: cluster 2023-10-09T04:37:10.420808+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:11.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:11 smithi153 bash[59565]: audit 2023-10-09T04:37:11.701056+0000 mon.a (mon.0) 1025 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:11.926 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:37:11.946 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:37:11.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:11.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:11.969 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:37:11.992 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:37:12.030 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:37:12.046 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:11 smithi082 bash[79733]: cluster 2023-10-09T04:37:10.420808+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:12.047 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:11 smithi082 bash[79733]: audit 2023-10-09T04:37:11.701056+0000 mon.a (mon.0) 1025 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:12.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:12.054 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:37:12.081 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:37:12.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:12.110 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:37:12.112 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.112 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.112 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.112 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.113 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.113 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.113 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.113 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:37:12.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:12.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:12.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:12.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:12.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:12.471 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:12.577 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:12.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:12.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:12.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:12.743 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:12.760 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_2' created 2023-10-09T04:37:12.847 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:12.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:12.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: cluster 2023-10-09T04:37:11.729022+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: cluster 2023-10-09T04:37:11.731421+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: audit 2023-10-09T04:37:11.734150+0000 mon.a (mon.0) 1026 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:12.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: cluster 2023-10-09T04:37:11.734249+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:37:12.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: cluster 2023-10-09T04:37:11.756427+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: cluster 2023-10-09T04:37:11.756466+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: audit 2023-10-09T04:37:11.902997+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.82:0/1458174032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:12.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:12 smithi153 bash[59565]: audit 2023-10-09T04:37:12.154874+0000 mon.a (mon.0) 1028 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:37:12.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:12.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: cluster 2023-10-09T04:37:11.729022+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: cluster 2023-10-09T04:37:11.731421+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: audit 2023-10-09T04:37:11.734150+0000 mon.a (mon.0) 1026 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: cluster 2023-10-09T04:37:11.734249+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: cluster 2023-10-09T04:37:11.756427+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: cluster 2023-10-09T04:37:11.756466+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: audit 2023-10-09T04:37:11.902997+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.82:0/1458174032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[77063]: audit 2023-10-09T04:37:12.154874+0000 mon.a (mon.0) 1028 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:37:12.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: cluster 2023-10-09T04:37:11.729022+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: cluster 2023-10-09T04:37:11.731421+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: audit 2023-10-09T04:37:11.734150+0000 mon.a (mon.0) 1026 : audit 1 from='client.? 172.21.15.82:0/1176547257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: cluster 2023-10-09T04:37:11.734249+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: cluster 2023-10-09T04:37:11.756427+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: cluster 2023-10-09T04:37:11.756466+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: audit 2023-10-09T04:37:11.902997+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.82:0/1458174032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:12.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:12 smithi082 bash[79733]: audit 2023-10-09T04:37:12.154874+0000 mon.a (mon.0) 1028 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:37:13.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:13.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:13.134 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:13.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:37:13.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:13.305 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:13.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:13.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:13.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:13.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:13.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:13.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:13.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.421188+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v683: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.747095+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.750899+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.750951+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.751074+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: audit 2023-10-09T04:37:12.751378+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.751458+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:37:13.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[77063]: cluster 2023-10-09T04:37:12.754620+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.421188+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v683: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.747095+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.750899+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.750951+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.751074+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: audit 2023-10-09T04:37:12.751378+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.751458+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:37:13.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:13 smithi082 bash[79733]: cluster 2023-10-09T04:37:12.754620+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:13.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:13.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:14.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:14.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.421188+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v683: 73 pgs: 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.747095+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.750899+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.750951+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.751074+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: audit 2023-10-09T04:37:12.751378+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.82:0/324679987' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:37:14.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.751458+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:37:14.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:13 smithi153 bash[59565]: cluster 2023-10-09T04:37:12.754620+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T04:37:14.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:14.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:14.335 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:14.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:14.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:14.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:14.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:14.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:14.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:14.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:14.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T04:37:14.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:14.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:14.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.751596+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:14.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.753008+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:14.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.755962+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:37:14.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.756363+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:14.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.756415+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:14.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: cluster 2023-10-09T04:37:13.756454+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:14.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:14 smithi153 bash[59565]: audit 2023-10-09T04:37:14.270720+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:14.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:14.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:15.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.018 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.751596+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.018 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.753008+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.755962+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.756363+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.756415+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: cluster 2023-10-09T04:37:13.756454+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[77063]: audit 2023-10-09T04:37:14.270720+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.751596+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.753008+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.755962+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.756363+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.756415+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.020 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: cluster 2023-10-09T04:37:13.756454+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T04:37:15.020 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:14 smithi082 bash[79733]: audit 2023-10-09T04:37:14.270720+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:15.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:15.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.173 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:15.234 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:15.234 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:15.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:15.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:15.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:15.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:15.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:15.580 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:15.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:15.715 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:15.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:15 smithi153 bash[59565]: cluster 2023-10-09T04:37:14.421740+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v686: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:15 smithi153 bash[59565]: cluster 2023-10-09T04:37:14.773216+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:15 smithi153 bash[59565]: cluster 2023-10-09T04:37:14.773549+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:15 smithi153 bash[59565]: cluster 2023-10-09T04:37:14.776361+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:37:15.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:15 smithi153 bash[59565]: cluster 2023-10-09T04:37:15.080335+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:15.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:15.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:15.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:16.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:16.103 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[77063]: cluster 2023-10-09T04:37:14.421740+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v686: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:16.103 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[77063]: cluster 2023-10-09T04:37:14.773216+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.103 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[77063]: cluster 2023-10-09T04:37:14.773549+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.103 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[77063]: cluster 2023-10-09T04:37:14.776361+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:37:16.103 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[77063]: cluster 2023-10-09T04:37:15.080335+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[79733]: cluster 2023-10-09T04:37:14.421740+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v686: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:16.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[79733]: cluster 2023-10-09T04:37:14.773216+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[79733]: cluster 2023-10-09T04:37:14.773549+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[79733]: cluster 2023-10-09T04:37:14.776361+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:37:16.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:15 smithi082 bash[79733]: cluster 2023-10-09T04:37:15.080335+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:16.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:16.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:16.255 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:16.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:16.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:16.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:16.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:16.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:16.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:16.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:37:16.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:16.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:16.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:16.921 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:16.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:16.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:16.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:16 smithi082 bash[77063]: cluster 2023-10-09T04:37:15.080368+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:16.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:16 smithi082 bash[79733]: cluster 2023-10-09T04:37:15.080368+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:17.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:17.038 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:16 smithi153 bash[59565]: cluster 2023-10-09T04:37:15.080368+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T04:37:17.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:17.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:17.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:17.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:17.319 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:17.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:17.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:17.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:17.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:17.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:17.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:17.829 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:17.859 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:17.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:17 smithi153 bash[59565]: cluster 2023-10-09T04:37:16.422738+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v688: 89 pgs: 11 creating+peering, 78 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:17.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:17 smithi082 bash[77063]: cluster 2023-10-09T04:37:16.422738+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v688: 89 pgs: 11 creating+peering, 78 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:17.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:17 smithi082 bash[79733]: cluster 2023-10-09T04:37:16.422738+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v688: 89 pgs: 11 creating+peering, 78 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:17.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:17.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:18.093 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:18.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:18.223 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:18.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:18.360 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:18.390 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:18.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:18.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:18.524 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:18.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:18.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:18.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:18.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:18.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:18.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:18.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:18.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:19.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:19.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:19.161 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:19.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:19.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:19.326 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:19.358 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:19.358 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:19.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:19.461 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:19.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:19.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:19.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:19.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:19.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:19] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.33.4" 2023-10-09T04:37:19.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:19.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:19.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:19.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:19.994 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:19.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:20.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:20.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:19 smithi153 bash[59565]: cluster 2023-10-09T04:37:18.423677+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:20.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:19 smithi082 bash[77063]: cluster 2023-10-09T04:37:18.423677+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:19 smithi082 bash[79733]: cluster 2023-10-09T04:37:18.423677+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:20.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:20.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:20.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:20.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:20.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:20.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:20.754 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:20.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:20.889 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:20.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:21.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:21.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:21.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:21.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:21.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:21.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:21.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:21.323 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:21.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:21.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:21.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:21.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:21.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:21.864 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:21 smithi082 bash[77063]: cluster 2023-10-09T04:37:20.424655+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:21.865 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:21 smithi082 bash[79733]: cluster 2023-10-09T04:37:20.424655+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:21.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:21.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:21.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:22.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:22.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:21 smithi153 bash[59565]: cluster 2023-10-09T04:37:20.424655+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:22.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:22.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:22.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:22.325 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:22.353 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:22.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:22.595 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:22.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:22.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:22.829 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:22.829 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:22.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:22.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:22.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:23.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:23.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:23.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:23.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:23.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:23.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:23.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:23.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:23.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:23.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:23.930 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:23.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:23.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:24.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:24.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:24.125 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:23 smithi082 bash[77063]: cluster 2023-10-09T04:37:22.425280+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:24.125 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:23 smithi082 bash[79733]: cluster 2023-10-09T04:37:22.425280+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:24.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:23 smithi153 bash[59565]: cluster 2023-10-09T04:37:22.425280+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:24.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:24.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:24.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:24.356 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:24.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:24.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:24.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:24.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:24.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:24.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:24.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:24.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:24.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:24.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:25.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:25.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:25.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:25.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:25.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:25.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:25.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:25.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:25.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:25.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:25.786 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:25.818 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:25.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:25 smithi082 bash[77063]: cluster 2023-10-09T04:37:24.425743+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:25.896 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:25 smithi082 bash[77063]: audit 2023-10-09T04:37:24.718124+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:25.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:25 smithi082 bash[79733]: cluster 2023-10-09T04:37:24.425743+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:25.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:25 smithi082 bash[79733]: audit 2023-10-09T04:37:24.718124+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:25.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:25.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:26.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:37:26.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:26.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:26.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:25 smithi153 bash[59565]: cluster 2023-10-09T04:37:24.425743+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:26.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:25 smithi153 bash[59565]: audit 2023-10-09T04:37:24.718124+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:26.201 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:26.224 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:26.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:26.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:26.467 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:26.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:26.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:26.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:26.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:26.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:26.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:26.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:26.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:26.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:26.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:27.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:27.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:27.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:27.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:27.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:27.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:27.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:27.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:27.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:27.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:27.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:27.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:27.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:27.741 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:37:27.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:27.786 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:27 smithi082 bash[77063]: cluster 2023-10-09T04:37:26.426364+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:27.786 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:27 smithi082 bash[77063]: audit 2023-10-09T04:37:26.813494+0000 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:27.786 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:27 smithi082 bash[79733]: cluster 2023-10-09T04:37:26.426364+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:27.786 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:27 smithi082 bash[79733]: audit 2023-10-09T04:37:26.813494+0000 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:27.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:27.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:27.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:27.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:28.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:28.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:28.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:27 smithi153 bash[59565]: cluster 2023-10-09T04:37:26.426364+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:28.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:27 smithi153 bash[59565]: audit 2023-10-09T04:37:26.813494+0000 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:28.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:28.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:28.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:28.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:28.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:28.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:28.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:28.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:28.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:28.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:28.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:28 smithi153 bash[59565]: cluster 2023-10-09T04:37:27.720157+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:28.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:28 smithi153 bash[59565]: cluster 2023-10-09T04:37:27.732488+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:28 smithi153 bash[59565]: audit 2023-10-09T04:37:27.735121+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:28.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:28 smithi153 bash[59565]: cluster 2023-10-09T04:37:27.735219+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:37:28.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:28 smithi153 bash[59565]: cluster 2023-10-09T04:37:27.736985+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:28.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[77063]: cluster 2023-10-09T04:37:27.720157+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[77063]: cluster 2023-10-09T04:37:27.732488+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[77063]: audit 2023-10-09T04:37:27.735121+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[77063]: cluster 2023-10-09T04:37:27.735219+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[77063]: cluster 2023-10-09T04:37:27.736985+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[79733]: cluster 2023-10-09T04:37:27.720157+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[79733]: cluster 2023-10-09T04:37:27.732488+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[79733]: audit 2023-10-09T04:37:27.735121+0000 mon.a (mon.0) 1037 : audit 1 from='client.? 172.21.15.82:0/3193828183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[79733]: cluster 2023-10-09T04:37:27.735219+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:37:28.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:28 smithi082 bash[79733]: cluster 2023-10-09T04:37:27.736985+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:28.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:29.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:29.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:29.144 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:29.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:29.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:29.385 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:29.416 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:29.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:29.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:29.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:29.673 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:29.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:29.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:29.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:29.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:29.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:30.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:30.052 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:29] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-09T04:37:30.053 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[77063]: cluster 2023-10-09T04:37:28.426966+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:30.053 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[77063]: cluster 2023-10-09T04:37:28.646992+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:30.053 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[77063]: audit 2023-10-09T04:37:29.270961+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:30.053 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[79733]: cluster 2023-10-09T04:37:28.426966+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:30.053 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[79733]: cluster 2023-10-09T04:37:28.646992+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:30.054 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:29 smithi082 bash[79733]: audit 2023-10-09T04:37:29.270961+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:30.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:30.162 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:29 smithi153 bash[59565]: cluster 2023-10-09T04:37:28.426966+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:37:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:29 smithi153 bash[59565]: cluster 2023-10-09T04:37:28.646992+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T04:37:30.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:29 smithi153 bash[59565]: audit 2023-10-09T04:37:29.270961+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:30.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:30.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:30.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:30.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:30.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:30.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:30.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:30.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:30.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:30.798 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:30.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:30.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:30.971 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:30.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:31.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:31.109 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:31.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:31.213 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:31.247 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:31.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:31.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:31.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:31.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:31.618 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:31.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:31.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:31.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:31.756 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:31 smithi153 bash[59565]: cluster 2023-10-09T04:37:30.427583+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:31.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:31.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:31.912 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:31.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:31 smithi082 bash[79733]: cluster 2023-10-09T04:37:30.427583+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:31.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:31 smithi082 bash[77063]: cluster 2023-10-09T04:37:30.427583+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:32.016 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:32.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:32.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:32.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:32.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.305 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:32.352 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:37:32.375 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:37:32.403 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:37:32.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:32.431 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:37:32.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:32.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:32.456 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:37:32.483 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:37:32.512 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:37:32.542 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:37:32.544 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.544 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.544 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.544 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.545 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.545 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.545 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.545 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:37:32.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:32.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:32.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:32.766 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T04:37:32.766 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:37:32.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:32.816 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[79733]: audit 2023-10-09T04:37:31.888510+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.816 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[79733]: audit 2023-10-09T04:37:32.325477+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.82:0/500191728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[79733]: audit 2023-10-09T04:37:32.526647+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.82:0/2725004255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[79733]: audit 2023-10-09T04:37:32.527237+0000 mon.a (mon.0) 1041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[77063]: audit 2023-10-09T04:37:31.888510+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[77063]: audit 2023-10-09T04:37:32.325477+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.82:0/500191728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[77063]: audit 2023-10-09T04:37:32.526647+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.82:0/2725004255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:32.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:32 smithi082 bash[77063]: audit 2023-10-09T04:37:32.527237+0000 mon.a (mon.0) 1041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:32.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:32.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:32.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:33.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:33.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:32 smithi153 bash[59565]: audit 2023-10-09T04:37:31.888510+0000 mon.a (mon.0) 1040 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:33.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:32 smithi153 bash[59565]: audit 2023-10-09T04:37:32.325477+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.82:0/500191728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:33.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:32 smithi153 bash[59565]: audit 2023-10-09T04:37:32.526647+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.82:0/2725004255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:33.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:32 smithi153 bash[59565]: audit 2023-10-09T04:37:32.527237+0000 mon.a (mon.0) 1041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:33.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:33.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:33.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:33.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:33.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:33.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:33.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:33.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:37:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:33.634 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:33.653 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:33.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:33.782 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:33.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:33.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:33.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: cluster 2023-10-09T04:37:32.427986+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:33.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: cluster 2023-10-09T04:37:32.749810+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: cluster 2023-10-09T04:37:32.754159+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: audit 2023-10-09T04:37:32.759550+0000 mon.a (mon.0) 1042 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: audit 2023-10-09T04:37:32.759681+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[77063]: cluster 2023-10-09T04:37:32.759891+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: cluster 2023-10-09T04:37:32.427986+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: cluster 2023-10-09T04:37:32.749810+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: cluster 2023-10-09T04:37:32.754159+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:33.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: audit 2023-10-09T04:37:32.759550+0000 mon.a (mon.0) 1042 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:37:33.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: audit 2023-10-09T04:37:32.759681+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:33.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:33 smithi082 bash[79733]: cluster 2023-10-09T04:37:32.759891+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:37:33.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:33.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:33.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:34.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: cluster 2023-10-09T04:37:32.427986+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: cluster 2023-10-09T04:37:32.749810+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: cluster 2023-10-09T04:37:32.754159+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: audit 2023-10-09T04:37:32.759550+0000 mon.a (mon.0) 1042 : audit 1 from='client.? 172.21.15.82:0/755532349' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: audit 2023-10-09T04:37:32.759681+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:34.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:33 smithi153 bash[59565]: cluster 2023-10-09T04:37:32.759891+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:37:34.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:34.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:34.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:34.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:34.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:34.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:34.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:34.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:34.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:34.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:34.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:34.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:34.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:34.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:34.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:34 smithi082 bash[77063]: cluster 2023-10-09T04:37:33.192672+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:34.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:34 smithi082 bash[79733]: cluster 2023-10-09T04:37:33.192672+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:34.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:34.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:35.085 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:37:35.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:37:35.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:35.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:34 smithi153 bash[59565]: cluster 2023-10-09T04:37:33.192672+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T04:37:35.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:35.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:35.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:35.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:35.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:35.608 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:35.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:35.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:35.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:35.788 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:35 smithi153 bash[59565]: cluster 2023-10-09T04:37:34.428681+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:35.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:35.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:37:35.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:35 smithi082 bash[77063]: cluster 2023-10-09T04:37:34.428681+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:35.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:35 smithi082 bash[79733]: cluster 2023-10-09T04:37:34.428681+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:35.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:35.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:36.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:36.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:36.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:36.125 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T04:37:36.127 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:37:36.128 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:37:36.162 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:37:36.162 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:37:36.162 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_155217 2023-10-09T04:37:36.163 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:37:36.163 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:37:36.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:36.316 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:36.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:36.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:36.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:36.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:36.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:36.724 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:36.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:36 smithi082 bash[77063]: audit 2023-10-09T04:37:36.704692+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:36.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:36 smithi082 bash[79733]: audit 2023-10-09T04:37:36.704692+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:36.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:36.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:37.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:37.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:37.166 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 16 2530 2514 156.579 157.125 0.0105931 0.00633462 2023-10-09T04:37:37.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:36 smithi153 bash[59565]: audit 2023-10-09T04:37:36.704692+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:37.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:37.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:37.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:37.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:37.544 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:37.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:37.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:37.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:37.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:37.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:37.960 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:37 smithi082 bash[77063]: cluster 2023-10-09T04:37:36.429410+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:37.961 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:37 smithi082 bash[79733]: cluster 2023-10-09T04:37:36.429410+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:37.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:37.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:38.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:37 smithi153 bash[59565]: cluster 2023-10-09T04:37:36.429410+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:37:38.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:38.166 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 16 5116 5100 159.089 161.625 0.0127604 0.00626233 2023-10-09T04:37:38.221 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:38.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:38.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:38.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:38.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:38.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:38.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:38.797 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:38.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:38.929 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:38.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:38.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:39.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:39.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:39.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:39.167 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 16 7687 7671 159.567 160.688 0.00881931 0.00625351 2023-10-09T04:37:39.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:39.299 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:39.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:39.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:39.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:39.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:39.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:39.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:39.690 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:39.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:39.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:39.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:39.856 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:39 smithi082 bash[77063]: cluster 2023-10-09T04:37:38.430075+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 164 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-10-09T04:37:39.857 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:39 smithi082 bash[79733]: cluster 2023-10-09T04:37:38.430075+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 164 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-10-09T04:37:39.857 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:39] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-09T04:37:39.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:39.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:39.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:39.981 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:39 smithi153 bash[59565]: cluster 2023-10-09T04:37:38.430075+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 164 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 MiB/s wr, 255 op/s 2023-10-09T04:37:39.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:40.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:40.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:40.167 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 16 9963 9947 155.239 142.25 0.00311991 0.00642773 2023-10-09T04:37:40.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:40.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:40.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:40.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:40.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:40.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:40.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:40.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:40.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:40.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:40.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:40.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:40.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:40.934 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:40.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:41.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:41.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:41.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:41.167 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 12348 12332 154.002 149.062 0.00540316 0.00648523 2023-10-09T04:37:41.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:41.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:41.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:41.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:41.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:41.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:41.706 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:41.727 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:41.728 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:41.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:41.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:41.853 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:41 smithi082 bash[77063]: cluster 2023-10-09T04:37:40.430884+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s 2023-10-09T04:37:41.854 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:41 smithi082 bash[79733]: cluster 2023-10-09T04:37:40.430884+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s 2023-10-09T04:37:41.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:41.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:42.014 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.023 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:41 smithi153 bash[59565]: cluster 2023-10-09T04:37:40.430884+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s 2023-10-09T04:37:42.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:42.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:42.167 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 16 14384 14368 149.545 127.25 0.00489344 0.00667597 2023-10-09T04:37:42.182 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:42.286 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:42.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:42.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:42.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:42.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:42.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:42.692 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:42.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:42.857 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:42.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:42.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:43.045 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:43.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:43.167 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 16025 16009 142.836 102.562 0.00320401 0.00697674 2023-10-09T04:37:43.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:43.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:43.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:43.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:43.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:43.532 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:37:43.532 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:37:43.532 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:37:43.532 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:37:43.533 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:37:43.540 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:43.563 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:43.666 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:43.691 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:43.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:43.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:43.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:43.880 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[77063]: cluster 2023-10-09T04:37:42.431393+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 531 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:37:43.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[77063]: audit 2023-10-09T04:37:43.169968+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:43.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[77063]: audit 2023-10-09T04:37:43.177674+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:43.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[77063]: audit 2023-10-09T04:37:43.223736+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:43.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[77063]: audit 2023-10-09T04:37:43.232110+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:43.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:43.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:43.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:44.057 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:44.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:43 smithi153 bash[59565]: cluster 2023-10-09T04:37:42.431393+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 531 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:37:44.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:43 smithi153 bash[59565]: audit 2023-10-09T04:37:43.169968+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:43 smithi153 bash[59565]: audit 2023-10-09T04:37:43.177674+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:43 smithi153 bash[59565]: audit 2023-10-09T04:37:43.223736+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:43 smithi153 bash[59565]: audit 2023-10-09T04:37:43.232110+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:44.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 17753 17737 138.483 108 0.0105754 0.00721351 2023-10-09T04:37:44.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:44.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[79733]: cluster 2023-10-09T04:37:42.431393+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 531 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:37:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[79733]: audit 2023-10-09T04:37:43.169968+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[79733]: audit 2023-10-09T04:37:43.177674+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[79733]: audit 2023-10-09T04:37:43.223736+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:43 smithi082 bash[79733]: audit 2023-10-09T04:37:43.232110+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:44.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:44.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:44.481 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:44.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:44.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:44.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:44.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:44.800 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:44 smithi153 bash[59565]: audit 2023-10-09T04:37:43.868169+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:44.800 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:44 smithi153 bash[59565]: audit 2023-10-09T04:37:43.869615+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:44.801 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:44 smithi153 bash[59565]: audit 2023-10-09T04:37:43.878936+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.801 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:44 smithi153 bash[59565]: audit 2023-10-09T04:37:44.271345+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:44.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:44.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:44.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:44.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[77063]: audit 2023-10-09T04:37:43.868169+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[77063]: audit 2023-10-09T04:37:43.869615+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[77063]: audit 2023-10-09T04:37:43.878936+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[77063]: audit 2023-10-09T04:37:44.271345+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:44.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[79733]: audit 2023-10-09T04:37:43.868169+0000 mon.a (mon.0) 1050 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[79733]: audit 2023-10-09T04:37:43.869615+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[79733]: audit 2023-10-09T04:37:43.878936+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:44 smithi082 bash[79733]: audit 2023-10-09T04:37:44.271345+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:45.002 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:45.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:45.139 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:45.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:45.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 19472 19456 135.034 107.438 0.00333333 0.00739471 2023-10-09T04:37:45.243 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:45.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:45.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:45.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:45.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:45.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:45.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:45.669 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:45.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:45.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:45.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:45.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:45.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:45.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:46.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:46.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 21355 21339 133.299 117.688 0.0123229 0.00749337 2023-10-09T04:37:46.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:46.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:45 smithi153 bash[59565]: cluster 2023-10-09T04:37:44.431872+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 834 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:37:46.199 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:46.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:45 smithi082 bash[77063]: cluster 2023-10-09T04:37:44.431872+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 834 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:37:46.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:45 smithi082 bash[79733]: cluster 2023-10-09T04:37:44.431872+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 834 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:37:46.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:46.433 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:46.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:46.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:46.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:46.600 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:46.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:46.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:46.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:46.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.865 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:46.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:46.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:46.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:47.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:47.124 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:47.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 16 23186 23170 131.584 114.438 0.00869154 0.00759316 2023-10-09T04:37:47.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:47.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:47.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:47.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.528 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:47.633 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:47.663 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:47.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:47.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:47.923 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:47.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:48.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:48.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 25127 25111 130.727 121.312 0.0110716 0.00764267 2023-10-09T04:37:48.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:47 smithi082 bash[77063]: cluster 2023-10-09T04:37:46.432593+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:37:48.168 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:47 smithi082 bash[79733]: cluster 2023-10-09T04:37:46.432593+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:37:48.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:47 smithi153 bash[59565]: cluster 2023-10-09T04:37:46.432593+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:37:48.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:48.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.324 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:48.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:48.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:48.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:37:48.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:48.614 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:48.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.750 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:48.857 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:48.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:48.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:48.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:49.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:49.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:49.151 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:49.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 26771 26755 128.575 102.75 0.00235294 0.00775206 2023-10-09T04:37:49.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:49.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:49.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:49.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:49.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:49.525 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:49.553 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:49.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:49.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:49.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:49.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:49.819 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:49.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:49.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:49.949 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:49 smithi082 bash[77063]: cluster 2023-10-09T04:37:48.433096+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:49.949 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:49] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.33.4" 2023-10-09T04:37:49.949 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:49 smithi082 bash[79733]: cluster 2023-10-09T04:37:48.433096+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:49.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:50.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:50.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:49 smithi153 bash[59565]: cluster 2023-10-09T04:37:48.433096+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:50.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:50.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 28128 28112 125.449 84.8125 0.00536343 0.00796546 2023-10-09T04:37:50.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:50.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:50.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:50.346 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:50.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:50.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:50.583 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:50.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:50.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:50.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:50.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:50.877 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:50.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:50.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:51.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:51.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:51.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 15 29871 29856 124.352 109 0.0099996 0.00803433 2023-10-09T04:37:51.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:51.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:51.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:51.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:51.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:51.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:51.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:51.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:51.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:51.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:51 smithi082 bash[77063]: cluster 2023-10-09T04:37:50.433742+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:51.927 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:51.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:52.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:52.036 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:51 smithi153 bash[59565]: cluster 2023-10-09T04:37:50.433742+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:52.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:52.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:52.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 31629 31613 123.443 109.812 0.0109936 0.00809368 2023-10-09T04:37:52.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:51 smithi082 bash[79733]: cluster 2023-10-09T04:37:50.433742+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:37:52.286 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:52.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:52.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:52.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:52.549 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:52.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:52.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:52.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:52.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:52.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:52.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:52.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:52.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:52.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:53.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:53.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:53.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 16 33292 33276 122.296 103.938 0.00372257 0.00817025 2023-10-09T04:37:53.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:53.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:53.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:53.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:53.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:53.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:53.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:53.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:53.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:53.741 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:53.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:53.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:53.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:53.900 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:53 smithi082 bash[77063]: cluster 2023-10-09T04:37:52.434206+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:37:53.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:53.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:54.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:54.038 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:54.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:54.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:53 smithi153 bash[59565]: cluster 2023-10-09T04:37:52.434206+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:37:54.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:53 smithi082 bash[79733]: cluster 2023-10-09T04:37:52.434206+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:37:54.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 34834 34818 120.856 96.375 0.00444489 0.00826711 2023-10-09T04:37:54.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:54.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:54.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:54.411 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:54.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:54.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:54.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:54.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:54.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:54.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:54.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:54.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:54.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:54.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:54.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:55.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:55.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 16 36236 36220 119.107 87.625 0.00973261 0.00838845 2023-10-09T04:37:55.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:55.343 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:55.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:55.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:55.509 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:55.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:55.672 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:37:55.746 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:55.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:55.886 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:55 smithi153 bash[59565]: cluster 2023-10-09T04:37:54.434811+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:37:55.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:55 smithi153 bash[59565]: audit 2023-10-09T04:37:54.766771+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.82:0/99675146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:55.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:55 smithi153 bash[59565]: audit 2023-10-09T04:37:54.767338+0000 mon.a (mon.0) 1054 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:55.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:55.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[77063]: cluster 2023-10-09T04:37:54.434811+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[77063]: audit 2023-10-09T04:37:54.766771+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.82:0/99675146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[77063]: audit 2023-10-09T04:37:54.767338+0000 mon.a (mon.0) 1054 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[79733]: cluster 2023-10-09T04:37:54.434811+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[79733]: audit 2023-10-09T04:37:54.766771+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.82:0/99675146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:55 smithi082 bash[79733]: audit 2023-10-09T04:37:54.767338+0000 mon.a (mon.0) 1054 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:56.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:56.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:56.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:56.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:37:56.167951+0000 min lat: 0.00131815 max lat: 0.138226 avg lat: 0.00850769 2023-10-09T04:37:56.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:37:56.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 16 37610 37594 117.445 85.875 0.0114577 0.00850769 2023-10-09T04:37:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:56.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:56.293 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:56.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.424 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:56.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:56.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:56.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:56.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:56.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:56.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.641322+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.641356+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.654744+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: audit 2023-10-09T04:37:55.657867+0000 mon.a (mon.0) 1057 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.658082+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.658977+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.662346+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:56 smithi153 bash[59565]: cluster 2023-10-09T04:37:55.662796+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.641322+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:56.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.641356+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T04:37:56.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.654744+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: audit 2023-10-09T04:37:55.657867+0000 mon.a (mon.0) 1057 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.658082+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.658977+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.662346+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[77063]: cluster 2023-10-09T04:37:55.662796+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.641322+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.641356+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.654744+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: audit 2023-10-09T04:37:55.657867+0000 mon.a (mon.0) 1057 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:56.930 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.658082+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:37:56.930 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.658977+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.930 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.662346+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.930 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:56 smithi082 bash[79733]: cluster 2023-10-09T04:37:55.662796+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T04:37:56.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:56.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:57.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:57.168 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 39147 39131 116.426 96.0625 0.0104185 0.00858347 2023-10-09T04:37:57.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:57.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:57.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:57.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:57.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:57.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:57.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:57.664 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:57 smithi082 bash[77063]: cluster 2023-10-09T04:37:56.435185+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s 2023-10-09T04:37:57.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:57.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.851 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:57 smithi153 bash[59565]: cluster 2023-10-09T04:37:56.435185+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s 2023-10-09T04:37:57.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:57.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:57.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:57 smithi082 bash[79733]: cluster 2023-10-09T04:37:56.435185+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 107 MiB/s wr, 1.70k op/s 2023-10-09T04:37:58.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:58.021 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:37:58.025 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:58.059 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:37:58.094 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:37:58.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:58.150 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:37:58.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 40237 40221 114.231 68.125 0.0127886 0.00874706 2023-10-09T04:37:58.190 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:37:58.230 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:37:58.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:58.267 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:37:58.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:58.309 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.326 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.327 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.327 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:37:58.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:58.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:58.408 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:37:58.413 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:37:58.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:58.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:58.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:58.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:58.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:58.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:58.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:58.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:58 smithi082 bash[77063]: audit 2023-10-09T04:37:57.968315+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.82:0/3909100013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:58.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:58 smithi082 bash[79733]: audit 2023-10-09T04:37:57.968315+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.82:0/3909100013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:59.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:59.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:58 smithi153 bash[59565]: audit 2023-10-09T04:37:57.968315+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.82:0/3909100013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:59.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:59.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:59.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:59.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 16 41451 41435 112.563 75.875 0.0041987 0.00887829 2023-10-09T04:37:59.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:59.291 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:59.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:59.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:59.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:59.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:59.573 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:59.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:59.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:37:59.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:59 smithi153 bash[59565]: audit 2023-10-09T04:37:58.385296+0000 mgr.y (mgr.34107) 712 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:37:59.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:59 smithi153 bash[59565]: cluster 2023-10-09T04:37:58.436757+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:37:59.679 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:59 smithi153 bash[59565]: audit 2023-10-09T04:37:59.279399+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:59.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:59.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:59.816 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[77063]: audit 2023-10-09T04:37:58.385296+0000 mgr.y (mgr.34107) 712 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:37:59.816 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[77063]: cluster 2023-10-09T04:37:58.436757+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[77063]: audit 2023-10-09T04:37:59.279399+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[77063]: audit 2023-10-09T04:37:59.280420+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:37:59] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.33.4" 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[79733]: audit 2023-10-09T04:37:58.385296+0000 mgr.y (mgr.34107) 712 : audit 0 from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[79733]: cluster 2023-10-09T04:37:58.436757+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[79733]: audit 2023-10-09T04:37:59.279399+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:37:59.817 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:37:59 smithi082 bash[79733]: audit 2023-10-09T04:37:59.280420+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:59.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:59.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:59.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:37:59.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:00.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:00.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:00.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 42822 42806 111.444 85.6875 0.00747007 0.00896789 2023-10-09T04:38:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:37:59 smithi153 bash[59565]: audit 2023-10-09T04:37:59.280420+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:00.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:00.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:00.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:00.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:00.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:00.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:00.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:00.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:00.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:00.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:00.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:00.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:00.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:00.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:01.010 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:01.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:01.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:01.137 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:01.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:01.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 43752 43736 109.311 58.125 0.0155755 0.00914289 2023-10-09T04:38:01.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:01.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:01.298 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:01.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:01.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:01.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:01.567 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:01.638 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":711,"stamp":"2023-10-09T04:37:56.434984+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2186876851,"num_objects":33324,"num_object_clones":0,"num_object_copies":99972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33324,"num_whiteouts":0,"num_read":6301,"num_read_kb":12735,"num_write":34402,"num_write_kb":2146103,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34466,"ondisk_log_size":34466,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8895884,"kb_used_data":6445772,"kb_used_omap":46,"kb_used_meta":2450001,"kb_avail":741065332,"statfs":{"total":767960285184,"available":758850899968,"internally_reserved":0,"allocated":6600470528,"data_stored":6595307716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47544,"internal_metadata":2508801608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,3,4,7],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1117257728,"num_objects":17048,"num_object_clones":0,"num_object_copies":51144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17048,"num_write_kb":1091072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002942"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":158,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737740+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:37:27.737740+0000","last_peered":"2023-10-09T04:37:27.737740+0000","last_clean":"2023-10-09T04:37:27.737740+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:37:27.737740+0000","last_undegraded":"2023-10-09T04:37:27.737740+0000","last_fullsized":"2023-10-09T04:37:27.737740+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.955620+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:37:27.955620+0000","last_peered":"2023-10-09T04:37:27.955620+0000","last_clean":"2023-10-09T04:37:27.955620+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:37:27.955620+0000","last_undegraded":"2023-10-09T04:37:27.955620+0000","last_fullsized":"2023-10-09T04:37:27.955620+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":159,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.951305+0000","last_change":"2023-10-09T04:36:24.634063+0000","last_active":"2023-10-09T04:37:28.951305+0000","last_peered":"2023-10-09T04:37:28.951305+0000","last_clean":"2023-10-09T04:37:28.951305+0000","last_became_active":"2023-10-09T04:36:24.633693+0000","last_became_peered":"2023-10-09T04:36:24.633693+0000","last_unstale":"2023-10-09T04:37:28.951305+0000","last_undegraded":"2023-10-09T04:37:28.951305+0000","last_fullsized":"2023-10-09T04:37:28.951305+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:34.420556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":131,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.734057+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:37:27.734057+0000","last_peered":"2023-10-09T04:37:27.734057+0000","last_clean":"2023-10-09T04:37:27.734057+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:37:27.734057+0000","last_undegraded":"2023-10-09T04:37:27.734057+0000","last_fullsized":"2023-10-09T04:37:27.734057+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":153,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733395+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:37:27.733395+0000","last_peered":"2023-10-09T04:37:27.733395+0000","last_clean":"2023-10-09T04:37:27.733395+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:37:27.733395+0000","last_undegraded":"2023-10-09T04:37:27.733395+0000","last_fullsized":"2023-10-09T04:37:27.733395+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":133,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952936+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:37:28.952936+0000","last_peered":"2023-10-09T04:37:28.952936+0000","last_clean":"2023-10-09T04:37:28.952936+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:37:28.952936+0000","last_undegraded":"2023-10-09T04:37:28.952936+0000","last_fullsized":"2023-10-09T04:37:28.952936+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":145,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.735902+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:37:27.735902+0000","last_peered":"2023-10-09T04:37:27.735902+0000","last_clean":"2023-10-09T04:37:27.735902+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:37:27.735902+0000","last_undegraded":"2023-10-09T04:37:27.735902+0000","last_fullsized":"2023-10-09T04:37:27.735902+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.951844+0000","last_change":"2023-10-09T04:34:09.918534+0000","last_active":"2023-10-09T04:37:28.951844+0000","last_peered":"2023-10-09T04:37:28.951844+0000","last_clean":"2023-10-09T04:37:28.951844+0000","last_became_active":"2023-10-09T04:34:09.918013+0000","last_became_peered":"2023-10-09T04:34:09.918013+0000","last_unstale":"2023-10-09T04:37:28.951844+0000","last_undegraded":"2023-10-09T04:37:28.951844+0000","last_fullsized":"2023-10-09T04:37:28.951844+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:00.542758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.742135+0000","last_change":"2023-10-09T04:34:36.648444+0000","last_active":"2023-10-09T04:37:27.742135+0000","last_peered":"2023-10-09T04:37:27.742135+0000","last_clean":"2023-10-09T04:37:27.742135+0000","last_became_active":"2023-10-09T04:34:36.647994+0000","last_became_peered":"2023-10-09T04:34:36.647994+0000","last_unstale":"2023-10-09T04:37:27.742135+0000","last_undegraded":"2023-10-09T04:37:27.742135+0000","last_fullsized":"2023-10-09T04:37:27.742135+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":140,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651197+0000","last_change":"2023-10-09T04:34:09.912750+0000","last_active":"2023-10-09T04:37:28.651197+0000","last_peered":"2023-10-09T04:37:28.651197+0000","last_clean":"2023-10-09T04:37:28.651197+0000","last_became_active":"2023-10-09T04:34:09.908040+0000","last_became_peered":"2023-10-09T04:34:09.908040+0000","last_unstale":"2023-10-09T04:37:28.651197+0000","last_undegraded":"2023-10-09T04:37:28.651197+0000","last_fullsized":"2023-10-09T04:37:28.651197+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":146,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651918+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:37:28.651918+0000","last_peered":"2023-10-09T04:37:28.651918+0000","last_clean":"2023-10-09T04:37:28.651918+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:37:28.651918+0000","last_undegraded":"2023-10-09T04:37:28.651918+0000","last_fullsized":"2023-10-09T04:37:28.651918+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.951736+0000","last_change":"2023-10-09T04:34:10.030630+0000","last_active":"2023-10-09T04:37:28.951736+0000","last_peered":"2023-10-09T04:37:28.951736+0000","last_clean":"2023-10-09T04:37:28.951736+0000","last_became_active":"2023-10-09T04:34:10.030403+0000","last_became_peered":"2023-10-09T04:34:10.030403+0000","last_unstale":"2023-10-09T04:37:28.951736+0000","last_undegraded":"2023-10-09T04:37:28.951736+0000","last_fullsized":"2023-10-09T04:37:28.951736+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:53.085670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":164,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738312+0000","last_change":"2023-10-09T04:34:09.968067+0000","last_active":"2023-10-09T04:37:27.738312+0000","last_peered":"2023-10-09T04:37:27.738312+0000","last_clean":"2023-10-09T04:37:27.738312+0000","last_became_active":"2023-10-09T04:34:09.967671+0000","last_became_peered":"2023-10-09T04:34:09.967671+0000","last_unstale":"2023-10-09T04:37:27.738312+0000","last_undegraded":"2023-10-09T04:37:27.738312+0000","last_fullsized":"2023-10-09T04:37:27.738312+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738535+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:37:27.738535+0000","last_peered":"2023-10-09T04:37:27.738535+0000","last_clean":"2023-10-09T04:37:27.738535+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:37:27.738535+0000","last_undegraded":"2023-10-09T04:37:27.738535+0000","last_fullsized":"2023-10-09T04:37:27.738535+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":145,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651982+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:37:28.651982+0000","last_peered":"2023-10-09T04:37:28.651982+0000","last_clean":"2023-10-09T04:37:28.651982+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:37:28.651982+0000","last_undegraded":"2023-10-09T04:37:28.651982+0000","last_fullsized":"2023-10-09T04:37:28.651982+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":138,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952133+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:37:28.952133+0000","last_peered":"2023-10-09T04:37:28.952133+0000","last_clean":"2023-10-09T04:37:28.952133+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:37:28.952133+0000","last_undegraded":"2023-10-09T04:37:28.952133+0000","last_fullsized":"2023-10-09T04:37:28.952133+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":130,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733667+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:37:27.733667+0000","last_peered":"2023-10-09T04:37:27.733667+0000","last_clean":"2023-10-09T04:37:27.733667+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:37:27.733667+0000","last_undegraded":"2023-10-09T04:37:27.733667+0000","last_fullsized":"2023-10-09T04:37:27.733667+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.742090+0000","last_change":"2023-10-09T04:34:36.646566+0000","last_active":"2023-10-09T04:37:27.742090+0000","last_peered":"2023-10-09T04:37:27.742090+0000","last_clean":"2023-10-09T04:37:27.742090+0000","last_became_active":"2023-10-09T04:34:36.646266+0000","last_became_peered":"2023-10-09T04:34:36.646266+0000","last_unstale":"2023-10-09T04:37:27.742090+0000","last_undegraded":"2023-10-09T04:37:27.742090+0000","last_fullsized":"2023-10-09T04:37:27.742090+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651169+0000","last_change":"2023-10-09T04:34:09.907681+0000","last_active":"2023-10-09T04:37:28.651169+0000","last_peered":"2023-10-09T04:37:28.651169+0000","last_clean":"2023-10-09T04:37:28.651169+0000","last_became_active":"2023-10-09T04:34:09.907156+0000","last_became_peered":"2023-10-09T04:34:09.907156+0000","last_unstale":"2023-10-09T04:37:28.651169+0000","last_undegraded":"2023-10-09T04:37:28.651169+0000","last_fullsized":"2023-10-09T04:37:28.651169+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733968+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:37:27.733968+0000","last_peered":"2023-10-09T04:37:27.733968+0000","last_clean":"2023-10-09T04:37:27.733968+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:37:27.733968+0000","last_undegraded":"2023-10-09T04:37:27.733968+0000","last_fullsized":"2023-10-09T04:37:27.733968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952162+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:37:28.952162+0000","last_peered":"2023-10-09T04:37:28.952162+0000","last_clean":"2023-10-09T04:37:28.952162+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:37:28.952162+0000","last_undegraded":"2023-10-09T04:37:28.952162+0000","last_fullsized":"2023-10-09T04:37:28.952162+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":135,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651839+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:37:28.651839+0000","last_peered":"2023-10-09T04:37:28.651839+0000","last_clean":"2023-10-09T04:37:28.651839+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:37:28.651839+0000","last_undegraded":"2023-10-09T04:37:28.651839+0000","last_fullsized":"2023-10-09T04:37:28.651839+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"177'2033","reported_seq":2046,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.674998+0000","last_change":"2023-10-09T04:37:13.779184+0000","last_active":"2023-10-09T04:37:53.674998+0000","last_peered":"2023-10-09T04:37:53.674998+0000","last_clean":"2023-10-09T04:37:53.674998+0000","last_became_active":"2023-10-09T04:37:13.778902+0000","last_became_peered":"2023-10-09T04:37:13.778902+0000","last_unstale":"2023-10-09T04:37:53.674998+0000","last_undegraded":"2023-10-09T04:37:53.674998+0000","last_fullsized":"2023-10-09T04:37:53.674998+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2033,"log_dups_size":0,"ondisk_log_size":2033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:15.517709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133234688,"num_objects":2033,"num_object_clones":0,"num_object_copies":6099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2033,"num_write_kb":130112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":144,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.652254+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:37:28.652254+0000","last_peered":"2023-10-09T04:37:28.652254+0000","last_clean":"2023-10-09T04:37:28.652254+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:37:28.652254+0000","last_undegraded":"2023-10-09T04:37:28.652254+0000","last_fullsized":"2023-10-09T04:37:28.652254+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":91,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.741666+0000","last_change":"2023-10-09T04:34:36.641326+0000","last_active":"2023-10-09T04:37:27.741666+0000","last_peered":"2023-10-09T04:37:27.741666+0000","last_clean":"2023-10-09T04:37:27.741666+0000","last_became_active":"2023-10-09T04:34:36.639824+0000","last_became_peered":"2023-10-09T04:34:36.639824+0000","last_unstale":"2023-10-09T04:37:27.741666+0000","last_undegraded":"2023-10-09T04:37:27.741666+0000","last_fullsized":"2023-10-09T04:37:27.741666+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"177'2153","reported_seq":2166,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.104678+0000","last_change":"2023-10-09T04:37:13.765303+0000","last_active":"2023-10-09T04:37:53.104678+0000","last_peered":"2023-10-09T04:37:53.104678+0000","last_clean":"2023-10-09T04:37:53.104678+0000","last_became_active":"2023-10-09T04:37:13.764983+0000","last_became_peered":"2023-10-09T04:37:13.764983+0000","last_unstale":"2023-10-09T04:37:53.104678+0000","last_undegraded":"2023-10-09T04:37:53.104678+0000","last_fullsized":"2023-10-09T04:37:53.104678+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2153,"log_dups_size":0,"ondisk_log_size":2153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.571025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141099008,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2153,"num_write_kb":137792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":171,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.650888+0000","last_change":"2023-10-09T04:36:24.678230+0000","last_active":"2023-10-09T04:37:28.650888+0000","last_peered":"2023-10-09T04:37:28.650888+0000","last_clean":"2023-10-09T04:37:28.650888+0000","last_became_active":"2023-10-09T04:36:24.633399+0000","last_became_peered":"2023-10-09T04:36:24.633399+0000","last_unstale":"2023-10-09T04:37:28.650888+0000","last_undegraded":"2023-10-09T04:37:28.650888+0000","last_fullsized":"2023-10-09T04:37:28.650888+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":156,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737851+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:37:27.737851+0000","last_peered":"2023-10-09T04:37:27.737851+0000","last_clean":"2023-10-09T04:37:27.737851+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:37:27.737851+0000","last_undegraded":"2023-10-09T04:37:27.737851+0000","last_fullsized":"2023-10-09T04:37:27.737851+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"177'1897","reported_seq":1910,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:51.585324+0000","last_change":"2023-10-09T04:37:13.775923+0000","last_active":"2023-10-09T04:37:51.585324+0000","last_peered":"2023-10-09T04:37:51.585324+0000","last_clean":"2023-10-09T04:37:51.585324+0000","last_became_active":"2023-10-09T04:37:13.775583+0000","last_became_peered":"2023-10-09T04:37:13.775583+0000","last_unstale":"2023-10-09T04:37:51.585324+0000","last_undegraded":"2023-10-09T04:37:51.585324+0000","last_fullsized":"2023-10-09T04:37:51.585324+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":1897,"log_dups_size":0,"ondisk_log_size":1897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:55.486175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124321792,"num_objects":1897,"num_object_clones":0,"num_object_copies":5691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1897,"num_write_kb":121408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":155,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651671+0000","last_change":"2023-10-09T04:34:36.648398+0000","last_active":"2023-10-09T04:37:28.651671+0000","last_peered":"2023-10-09T04:37:28.651671+0000","last_clean":"2023-10-09T04:37:28.651671+0000","last_became_active":"2023-10-09T04:34:36.647896+0000","last_became_peered":"2023-10-09T04:34:36.647896+0000","last_unstale":"2023-10-09T04:37:28.651671+0000","last_undegraded":"2023-10-09T04:37:28.651671+0000","last_fullsized":"2023-10-09T04:37:28.651671+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952131+0000","last_change":"2023-10-09T04:34:09.983232+0000","last_active":"2023-10-09T04:37:28.952131+0000","last_peered":"2023-10-09T04:37:28.952131+0000","last_clean":"2023-10-09T04:37:28.952131+0000","last_became_active":"2023-10-09T04:34:09.976353+0000","last_became_peered":"2023-10-09T04:34:09.976353+0000","last_unstale":"2023-10-09T04:37:28.952131+0000","last_undegraded":"2023-10-09T04:37:28.952131+0000","last_fullsized":"2023-10-09T04:37:28.952131+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"177'2095","reported_seq":2108,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.131826+0000","last_change":"2023-10-09T04:37:13.774933+0000","last_active":"2023-10-09T04:37:53.131826+0000","last_peered":"2023-10-09T04:37:53.131826+0000","last_clean":"2023-10-09T04:37:53.131826+0000","last_became_active":"2023-10-09T04:37:13.774485+0000","last_became_peered":"2023-10-09T04:37:13.774485+0000","last_unstale":"2023-10-09T04:37:53.131826+0000","last_undegraded":"2023-10-09T04:37:53.131826+0000","last_fullsized":"2023-10-09T04:37:53.131826+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2095,"log_dups_size":0,"ondisk_log_size":2095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.541922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137297920,"num_objects":2095,"num_object_clones":0,"num_object_copies":6285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2095,"num_write_kb":134080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.741876+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:37:27.741876+0000","last_peered":"2023-10-09T04:37:27.741876+0000","last_clean":"2023-10-09T04:37:27.741876+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:37:27.741876+0000","last_undegraded":"2023-10-09T04:37:27.741876+0000","last_fullsized":"2023-10-09T04:37:27.741876+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733545+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:37:27.733545+0000","last_peered":"2023-10-09T04:37:27.733545+0000","last_clean":"2023-10-09T04:37:27.733545+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:37:27.733545+0000","last_undegraded":"2023-10-09T04:37:27.733545+0000","last_fullsized":"2023-10-09T04:37:27.733545+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"177'1919","reported_seq":1932,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:51.570111+0000","last_change":"2023-10-09T04:37:13.769259+0000","last_active":"2023-10-09T04:37:51.570111+0000","last_peered":"2023-10-09T04:37:51.570111+0000","last_clean":"2023-10-09T04:37:51.570111+0000","last_became_active":"2023-10-09T04:37:13.768831+0000","last_became_peered":"2023-10-09T04:37:13.768831+0000","last_unstale":"2023-10-09T04:37:51.570111+0000","last_undegraded":"2023-10-09T04:37:51.570111+0000","last_fullsized":"2023-10-09T04:37:51.570111+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":1919,"log_dups_size":0,"ondisk_log_size":1919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:41.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125763584,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1919,"num_write_kb":122816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756457+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:37:32.756457+0000","last_peered":"2023-10-09T04:37:32.756457+0000","last_clean":"2023-10-09T04:37:32.756457+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:37:32.756457+0000","last_undegraded":"2023-10-09T04:37:32.756457+0000","last_fullsized":"2023-10-09T04:37:32.756457+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738471+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:37:27.738471+0000","last_peered":"2023-10-09T04:37:27.738471+0000","last_clean":"2023-10-09T04:37:27.738471+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:37:27.738471+0000","last_undegraded":"2023-10-09T04:37:27.738471+0000","last_fullsized":"2023-10-09T04:37:27.738471+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"177'2122","reported_seq":2135,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.112893+0000","last_change":"2023-10-09T04:37:13.764469+0000","last_active":"2023-10-09T04:37:53.112893+0000","last_peered":"2023-10-09T04:37:53.112893+0000","last_clean":"2023-10-09T04:37:53.112893+0000","last_became_active":"2023-10-09T04:37:13.763760+0000","last_became_peered":"2023-10-09T04:37:13.763760+0000","last_unstale":"2023-10-09T04:37:53.112893+0000","last_undegraded":"2023-10-09T04:37:53.112893+0000","last_fullsized":"2023-10-09T04:37:53.112893+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2122,"log_dups_size":0,"ondisk_log_size":2122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:08.322588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2122,"num_write_kb":135808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952147+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:37:28.952147+0000","last_peered":"2023-10-09T04:37:28.952147+0000","last_clean":"2023-10-09T04:37:28.952147+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:37:28.952147+0000","last_undegraded":"2023-10-09T04:37:28.952147+0000","last_fullsized":"2023-10-09T04:37:28.952147+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738290+0000","last_change":"2023-10-09T04:36:24.632828+0000","last_active":"2023-10-09T04:37:27.738290+0000","last_peered":"2023-10-09T04:37:27.738290+0000","last_clean":"2023-10-09T04:37:27.738290+0000","last_became_active":"2023-10-09T04:36:24.632286+0000","last_became_peered":"2023-10-09T04:36:24.632286+0000","last_unstale":"2023-10-09T04:37:27.738290+0000","last_undegraded":"2023-10-09T04:37:27.738290+0000","last_fullsized":"2023-10-09T04:37:27.738290+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"177'1996","reported_seq":2008,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.019692+0000","last_change":"2023-10-09T04:37:13.777270+0000","last_active":"2023-10-09T04:37:53.019692+0000","last_peered":"2023-10-09T04:37:53.019692+0000","last_clean":"2023-10-09T04:37:53.019692+0000","last_became_active":"2023-10-09T04:37:13.776717+0000","last_became_peered":"2023-10-09T04:37:13.776717+0000","last_unstale":"2023-10-09T04:37:53.019692+0000","last_undegraded":"2023-10-09T04:37:53.019692+0000","last_fullsized":"2023-10-09T04:37:53.019692+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":1996,"log_dups_size":0,"ondisk_log_size":1996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130809856,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1996,"num_write_kb":127744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737707+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:37:27.737707+0000","last_peered":"2023-10-09T04:37:27.737707+0000","last_clean":"2023-10-09T04:37:27.737707+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:37:27.737707+0000","last_undegraded":"2023-10-09T04:37:27.737707+0000","last_fullsized":"2023-10-09T04:37:27.737707+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738150+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:37:27.738150+0000","last_peered":"2023-10-09T04:37:27.738150+0000","last_clean":"2023-10-09T04:37:27.738150+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:37:27.738150+0000","last_undegraded":"2023-10-09T04:37:27.738150+0000","last_fullsized":"2023-10-09T04:37:27.738150+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"177'2023","reported_seq":2035,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.011793+0000","last_change":"2023-10-09T04:37:13.778215+0000","last_active":"2023-10-09T04:37:53.011793+0000","last_peered":"2023-10-09T04:37:53.011793+0000","last_clean":"2023-10-09T04:37:53.011793+0000","last_became_active":"2023-10-09T04:37:13.777797+0000","last_became_peered":"2023-10-09T04:37:13.777797+0000","last_unstale":"2023-10-09T04:37:53.011793+0000","last_undegraded":"2023-10-09T04:37:53.011793+0000","last_fullsized":"2023-10-09T04:37:53.011793+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2023,"log_dups_size":0,"ondisk_log_size":2023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2023,"num_write_kb":129472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":140,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.529334+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:37:34.529334+0000","last_peered":"2023-10-09T04:37:34.529334+0000","last_clean":"2023-10-09T04:37:34.529334+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:37:34.529334+0000","last_undegraded":"2023-10-09T04:37:34.529334+0000","last_fullsized":"2023-10-09T04:37:34.529334+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":126,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952442+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:37:28.952442+0000","last_peered":"2023-10-09T04:37:28.952442+0000","last_clean":"2023-10-09T04:37:28.952442+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:37:28.952442+0000","last_undegraded":"2023-10-09T04:37:28.952442+0000","last_fullsized":"2023-10-09T04:37:28.952442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"177'2271","reported_seq":2281,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.219207+0000","last_change":"2023-10-09T04:37:13.778496+0000","last_active":"2023-10-09T04:37:55.219207+0000","last_peered":"2023-10-09T04:37:55.219207+0000","last_clean":"2023-10-09T04:37:55.219207+0000","last_became_active":"2023-10-09T04:37:13.777643+0000","last_became_peered":"2023-10-09T04:37:13.777643+0000","last_unstale":"2023-10-09T04:37:55.219207+0000","last_undegraded":"2023-10-09T04:37:55.219207+0000","last_fullsized":"2023-10-09T04:37:55.219207+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2271,"log_dups_size":0,"ondisk_log_size":2271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148832256,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2271,"num_write_kb":145344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":552,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952286+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:37:28.952286+0000","last_peered":"2023-10-09T04:37:28.952286+0000","last_clean":"2023-10-09T04:37:28.952286+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:37:28.952286+0000","last_undegraded":"2023-10-09T04:37:28.952286+0000","last_fullsized":"2023-10-09T04:37:28.952286+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.742256+0000","last_change":"2023-10-09T04:34:36.645924+0000","last_active":"2023-10-09T04:37:27.742256+0000","last_peered":"2023-10-09T04:37:27.742256+0000","last_clean":"2023-10-09T04:37:27.742256+0000","last_became_active":"2023-10-09T04:34:36.645572+0000","last_became_peered":"2023-10-09T04:34:36.645572+0000","last_unstale":"2023-10-09T04:37:27.742256+0000","last_undegraded":"2023-10-09T04:37:27.742256+0000","last_fullsized":"2023-10-09T04:37:27.742256+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.951774+0000","last_change":"2023-10-09T04:34:36.642430+0000","last_active":"2023-10-09T04:37:28.951774+0000","last_peered":"2023-10-09T04:37:28.951774+0000","last_clean":"2023-10-09T04:37:28.951774+0000","last_became_active":"2023-10-09T04:34:36.641942+0000","last_became_peered":"2023-10-09T04:34:36.641942+0000","last_unstale":"2023-10-09T04:37:28.951774+0000","last_undegraded":"2023-10-09T04:37:28.951774+0000","last_fullsized":"2023-10-09T04:37:28.951774+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:29.110665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"177'2109","reported_seq":2121,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.017855+0000","last_change":"2023-10-09T04:37:13.778105+0000","last_active":"2023-10-09T04:37:53.017855+0000","last_peered":"2023-10-09T04:37:53.017855+0000","last_clean":"2023-10-09T04:37:53.017855+0000","last_became_active":"2023-10-09T04:37:13.777582+0000","last_became_peered":"2023-10-09T04:37:13.777582+0000","last_unstale":"2023-10-09T04:37:53.017855+0000","last_undegraded":"2023-10-09T04:37:53.017855+0000","last_fullsized":"2023-10-09T04:37:53.017855+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2109,"log_dups_size":0,"ondisk_log_size":2109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2109,"num_write_kb":134976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":639,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.741657+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:37:27.741657+0000","last_peered":"2023-10-09T04:37:27.741657+0000","last_clean":"2023-10-09T04:37:27.741657+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:37:27.741657+0000","last_undegraded":"2023-10-09T04:37:27.741657+0000","last_fullsized":"2023-10-09T04:37:27.741657+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":157,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.738465+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:37:27.738465+0000","last_peered":"2023-10-09T04:37:27.738465+0000","last_clean":"2023-10-09T04:37:27.738465+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:37:27.738465+0000","last_undegraded":"2023-10-09T04:37:27.738465+0000","last_fullsized":"2023-10-09T04:37:27.738465+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952045+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:37:28.952045+0000","last_peered":"2023-10-09T04:37:28.952045+0000","last_clean":"2023-10-09T04:37:28.952045+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:37:28.952045+0000","last_undegraded":"2023-10-09T04:37:28.952045+0000","last_fullsized":"2023-10-09T04:37:28.952045+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"177'1952","reported_seq":1964,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:51.578213+0000","last_change":"2023-10-09T04:37:13.775171+0000","last_active":"2023-10-09T04:37:51.578213+0000","last_peered":"2023-10-09T04:37:51.578213+0000","last_clean":"2023-10-09T04:37:51.578213+0000","last_became_active":"2023-10-09T04:37:13.774832+0000","last_became_peered":"2023-10-09T04:37:13.774832+0000","last_unstale":"2023-10-09T04:37:51.578213+0000","last_undegraded":"2023-10-09T04:37:51.578213+0000","last_fullsized":"2023-10-09T04:37:51.578213+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":1952,"log_dups_size":0,"ondisk_log_size":1952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:58.564050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127926272,"num_objects":1952,"num_object_clones":0,"num_object_copies":5856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1952,"num_write_kb":124928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1288,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.854017+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:37:34.854017+0000","last_peered":"2023-10-09T04:37:34.854017+0000","last_clean":"2023-10-09T04:37:34.854017+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:37:34.854017+0000","last_undegraded":"2023-10-09T04:37:34.854017+0000","last_fullsized":"2023-10-09T04:37:34.854017+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.951726+0000","last_change":"2023-10-09T04:34:09.901059+0000","last_active":"2023-10-09T04:37:28.951726+0000","last_peered":"2023-10-09T04:37:28.951726+0000","last_clean":"2023-10-09T04:37:28.951726+0000","last_became_active":"2023-10-09T04:34:09.900690+0000","last_became_peered":"2023-10-09T04:34:09.900690+0000","last_unstale":"2023-10-09T04:37:28.951726+0000","last_undegraded":"2023-10-09T04:37:28.951726+0000","last_fullsized":"2023-10-09T04:37:28.951726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:59.089033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":171,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952412+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:37:28.952412+0000","last_peered":"2023-10-09T04:37:28.952412+0000","last_clean":"2023-10-09T04:37:28.952412+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:37:28.952412+0000","last_undegraded":"2023-10-09T04:37:28.952412+0000","last_fullsized":"2023-10-09T04:37:28.952412+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"177'1854","reported_seq":1867,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:51.543435+0000","last_change":"2023-10-09T04:37:13.768144+0000","last_active":"2023-10-09T04:37:51.543435+0000","last_peered":"2023-10-09T04:37:51.543435+0000","last_clean":"2023-10-09T04:37:51.543435+0000","last_became_active":"2023-10-09T04:37:13.767811+0000","last_became_peered":"2023-10-09T04:37:13.767811+0000","last_unstale":"2023-10-09T04:37:51.543435+0000","last_undegraded":"2023-10-09T04:37:51.543435+0000","last_fullsized":"2023-10-09T04:37:51.543435+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":1854,"log_dups_size":0,"ondisk_log_size":1854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:06.150093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121503744,"num_objects":1854,"num_object_clones":0,"num_object_copies":5562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1854,"num_write_kb":118656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":621,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.741797+0000","last_change":"2023-10-09T04:34:36.645853+0000","last_active":"2023-10-09T04:37:27.741797+0000","last_peered":"2023-10-09T04:37:27.741797+0000","last_clean":"2023-10-09T04:37:27.741797+0000","last_became_active":"2023-10-09T04:34:36.644515+0000","last_became_peered":"2023-10-09T04:34:36.644515+0000","last_unstale":"2023-10-09T04:37:27.741797+0000","last_undegraded":"2023-10-09T04:37:27.741797+0000","last_fullsized":"2023-10-09T04:37:27.741797+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":394,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651409+0000","last_change":"2023-10-09T04:34:09.917744+0000","last_active":"2023-10-09T04:37:28.651409+0000","last_peered":"2023-10-09T04:37:28.651409+0000","last_clean":"2023-10-09T04:37:28.651409+0000","last_became_active":"2023-10-09T04:34:09.917258+0000","last_became_peered":"2023-10-09T04:34:09.917258+0000","last_unstale":"2023-10-09T04:37:28.651409+0000","last_undegraded":"2023-10-09T04:37:28.651409+0000","last_fullsized":"2023-10-09T04:37:28.651409+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":95,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.741582+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:37:27.741582+0000","last_peered":"2023-10-09T04:37:27.741582+0000","last_clean":"2023-10-09T04:37:27.741582+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:37:27.741582+0000","last_undegraded":"2023-10-09T04:37:27.741582+0000","last_fullsized":"2023-10-09T04:37:27.741582+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952713+0000","last_change":"2023-10-09T04:34:09.927188+0000","last_active":"2023-10-09T04:37:28.952713+0000","last_peered":"2023-10-09T04:37:28.952713+0000","last_clean":"2023-10-09T04:37:28.952713+0000","last_became_active":"2023-10-09T04:34:09.926564+0000","last_became_peered":"2023-10-09T04:34:09.926564+0000","last_unstale":"2023-10-09T04:37:28.952713+0000","last_undegraded":"2023-10-09T04:37:28.952713+0000","last_fullsized":"2023-10-09T04:37:28.952713+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"177'2307","reported_seq":2319,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.280558+0000","last_change":"2023-10-09T04:37:13.763373+0000","last_active":"2023-10-09T04:37:55.280558+0000","last_peered":"2023-10-09T04:37:55.280558+0000","last_clean":"2023-10-09T04:37:55.280558+0000","last_became_active":"2023-10-09T04:37:13.762984+0000","last_became_peered":"2023-10-09T04:37:13.762984+0000","last_unstale":"2023-10-09T04:37:55.280558+0000","last_undegraded":"2023-10-09T04:37:55.280558+0000","last_fullsized":"2023-10-09T04:37:55.280558+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2307,"log_dups_size":0,"ondisk_log_size":2307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151191552,"num_objects":2307,"num_object_clones":0,"num_object_copies":6921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2307,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":721,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651769+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:37:28.651769+0000","last_peered":"2023-10-09T04:37:28.651769+0000","last_clean":"2023-10-09T04:37:28.651769+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:37:28.651769+0000","last_undegraded":"2023-10-09T04:37:28.651769+0000","last_fullsized":"2023-10-09T04:37:28.651769+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":160,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733506+0000","last_change":"2023-10-09T04:36:24.632976+0000","last_active":"2023-10-09T04:37:27.733506+0000","last_peered":"2023-10-09T04:37:27.733506+0000","last_clean":"2023-10-09T04:37:27.733506+0000","last_became_active":"2023-10-09T04:36:24.632564+0000","last_became_peered":"2023-10-09T04:36:24.632564+0000","last_unstale":"2023-10-09T04:37:27.733506+0000","last_undegraded":"2023-10-09T04:37:27.733506+0000","last_fullsized":"2023-10-09T04:37:27.733506+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:11:16.527930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":146,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:31.584882+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:37:31.584882+0000","last_peered":"2023-10-09T04:37:31.584882+0000","last_clean":"2023-10-09T04:37:31.584882+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:37:31.584882+0000","last_undegraded":"2023-10-09T04:37:31.584882+0000","last_fullsized":"2023-10-09T04:37:31.584882+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"177'2312","reported_seq":2324,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:54.997841+0000","last_change":"2023-10-09T04:37:13.762982+0000","last_active":"2023-10-09T04:37:54.997841+0000","last_peered":"2023-10-09T04:37:54.997841+0000","last_clean":"2023-10-09T04:37:54.997841+0000","last_became_active":"2023-10-09T04:37:13.762606+0000","last_became_peered":"2023-10-09T04:37:13.762606+0000","last_unstale":"2023-10-09T04:37:54.997841+0000","last_undegraded":"2023-10-09T04:37:54.997841+0000","last_fullsized":"2023-10-09T04:37:54.997841+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2312,"log_dups_size":0,"ondisk_log_size":2312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151519232,"num_objects":2312,"num_object_clones":0,"num_object_copies":6936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2312,"num_write_kb":147968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":390,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651774+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:37:28.651774+0000","last_peered":"2023-10-09T04:37:28.651774+0000","last_clean":"2023-10-09T04:37:28.651774+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:37:28.651774+0000","last_undegraded":"2023-10-09T04:37:28.651774+0000","last_fullsized":"2023-10-09T04:37:28.651774+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952134+0000","last_change":"2023-10-09T04:34:36.642995+0000","last_active":"2023-10-09T04:37:28.952134+0000","last_peered":"2023-10-09T04:37:28.952134+0000","last_clean":"2023-10-09T04:37:28.952134+0000","last_became_active":"2023-10-09T04:34:36.642548+0000","last_became_peered":"2023-10-09T04:34:36.642548+0000","last_unstale":"2023-10-09T04:37:28.952134+0000","last_undegraded":"2023-10-09T04:37:28.952134+0000","last_fullsized":"2023-10-09T04:37:28.952134+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:13.000173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":131,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.734010+0000","last_change":"2023-10-09T04:34:36.642199+0000","last_active":"2023-10-09T04:37:27.734010+0000","last_peered":"2023-10-09T04:37:27.734010+0000","last_clean":"2023-10-09T04:37:27.734010+0000","last_became_active":"2023-10-09T04:34:36.641868+0000","last_became_peered":"2023-10-09T04:34:36.641868+0000","last_unstale":"2023-10-09T04:37:27.734010+0000","last_undegraded":"2023-10-09T04:37:27.734010+0000","last_fullsized":"2023-10-09T04:37:27.734010+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:45.514132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"177'2134","reported_seq":2147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.107280+0000","last_change":"2023-10-09T04:37:13.774838+0000","last_active":"2023-10-09T04:37:53.107280+0000","last_peered":"2023-10-09T04:37:53.107280+0000","last_clean":"2023-10-09T04:37:53.107280+0000","last_became_active":"2023-10-09T04:37:13.774474+0000","last_became_peered":"2023-10-09T04:37:13.774474+0000","last_unstale":"2023-10-09T04:37:53.107280+0000","last_undegraded":"2023-10-09T04:37:53.107280+0000","last_fullsized":"2023-10-09T04:37:53.107280+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2134,"log_dups_size":0,"ondisk_log_size":2134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:10.659935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2134,"num_write_kb":136576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":91,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.652049+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:37:28.652049+0000","last_peered":"2023-10-09T04:37:28.652049+0000","last_clean":"2023-10-09T04:37:28.652049+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:37:28.652049+0000","last_undegraded":"2023-10-09T04:37:28.652049+0000","last_fullsized":"2023-10-09T04:37:28.652049+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":417,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952286+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:37:28.952286+0000","last_peered":"2023-10-09T04:37:28.952286+0000","last_clean":"2023-10-09T04:37:28.952286+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:37:28.952286+0000","last_undegraded":"2023-10-09T04:37:28.952286+0000","last_fullsized":"2023-10-09T04:37:28.952286+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":130,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.651234+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:37:28.651234+0000","last_peered":"2023-10-09T04:37:28.651234+0000","last_clean":"2023-10-09T04:37:28.651234+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:37:28.651234+0000","last_undegraded":"2023-10-09T04:37:28.651234+0000","last_fullsized":"2023-10-09T04:37:28.651234+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"177'2119","reported_seq":2132,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.675448+0000","last_change":"2023-10-09T04:37:13.767844+0000","last_active":"2023-10-09T04:37:53.675448+0000","last_peered":"2023-10-09T04:37:53.675448+0000","last_clean":"2023-10-09T04:37:53.675448+0000","last_became_active":"2023-10-09T04:37:13.767535+0000","last_became_peered":"2023-10-09T04:37:13.767535+0000","last_unstale":"2023-10-09T04:37:53.675448+0000","last_undegraded":"2023-10-09T04:37:53.675448+0000","last_fullsized":"2023-10-09T04:37:53.675448+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":2119,"log_dups_size":0,"ondisk_log_size":2119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:31.233224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138870784,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2119,"num_write_kb":135616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":189,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.953039+0000","last_change":"2023-10-09T04:34:09.918839+0000","last_active":"2023-10-09T04:37:28.953039+0000","last_peered":"2023-10-09T04:37:28.953039+0000","last_clean":"2023-10-09T04:37:28.953039+0000","last_became_active":"2023-10-09T04:34:09.918201+0000","last_became_peered":"2023-10-09T04:34:09.918201+0000","last_unstale":"2023-10-09T04:37:28.953039+0000","last_undegraded":"2023-10-09T04:37:28.953039+0000","last_fullsized":"2023-10-09T04:37:28.953039+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":135,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:31.584844+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:37:31.584844+0000","last_peered":"2023-10-09T04:37:31.584844+0000","last_clean":"2023-10-09T04:37:31.584844+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:37:31.584844+0000","last_undegraded":"2023-10-09T04:37:31.584844+0000","last_fullsized":"2023-10-09T04:37:31.584844+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737734+0000","last_change":"2023-10-09T04:34:36.638499+0000","last_active":"2023-10-09T04:37:27.737734+0000","last_peered":"2023-10-09T04:37:27.737734+0000","last_clean":"2023-10-09T04:37:27.737734+0000","last_became_active":"2023-10-09T04:34:36.638089+0000","last_became_peered":"2023-10-09T04:34:36.638089+0000","last_unstale":"2023-10-09T04:37:27.737734+0000","last_undegraded":"2023-10-09T04:37:27.737734+0000","last_fullsized":"2023-10-09T04:37:27.737734+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":119,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.733930+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:37:27.733930+0000","last_peered":"2023-10-09T04:37:27.733930+0000","last_clean":"2023-10-09T04:37:27.733930+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:37:27.733930+0000","last_undegraded":"2023-10-09T04:37:27.733930+0000","last_fullsized":"2023-10-09T04:37:27.733930+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":148,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.955411+0000","last_change":"2023-10-09T04:34:36.644352+0000","last_active":"2023-10-09T04:37:27.955411+0000","last_peered":"2023-10-09T04:37:27.955411+0000","last_clean":"2023-10-09T04:37:27.955411+0000","last_became_active":"2023-10-09T04:34:36.643870+0000","last_became_peered":"2023-10-09T04:34:36.643870+0000","last_unstale":"2023-10-09T04:37:27.955411+0000","last_undegraded":"2023-10-09T04:37:27.955411+0000","last_fullsized":"2023-10-09T04:37:27.955411+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:32.488036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":102,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.742055+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:37:27.742055+0000","last_peered":"2023-10-09T04:37:27.742055+0000","last_clean":"2023-10-09T04:37:27.742055+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:37:27.742055+0000","last_undegraded":"2023-10-09T04:37:27.742055+0000","last_fullsized":"2023-10-09T04:37:27.742055+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737957+0000","last_change":"2023-10-09T04:36:24.632928+0000","last_active":"2023-10-09T04:37:27.737957+0000","last_peered":"2023-10-09T04:37:27.737957+0000","last_clean":"2023-10-09T04:37:27.737957+0000","last_became_active":"2023-10-09T04:36:24.632560+0000","last_became_peered":"2023-10-09T04:36:24.632560+0000","last_unstale":"2023-10-09T04:37:27.737957+0000","last_undegraded":"2023-10-09T04:37:27.737957+0000","last_fullsized":"2023-10-09T04:37:27.737957+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":95,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.742503+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:37:27.742503+0000","last_peered":"2023-10-09T04:37:27.742503+0000","last_clean":"2023-10-09T04:37:27.742503+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:37:27.742503+0000","last_undegraded":"2023-10-09T04:37:27.742503+0000","last_fullsized":"2023-10-09T04:37:27.742503+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737874+0000","last_change":"2023-10-09T04:36:24.632650+0000","last_active":"2023-10-09T04:37:27.737874+0000","last_peered":"2023-10-09T04:37:27.737874+0000","last_clean":"2023-10-09T04:37:27.737874+0000","last_became_active":"2023-10-09T04:36:24.632235+0000","last_became_peered":"2023-10-09T04:36:24.632235+0000","last_unstale":"2023-10-09T04:37:27.737874+0000","last_undegraded":"2023-10-09T04:37:27.737874+0000","last_fullsized":"2023-10-09T04:37:27.737874+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":64,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:28.952442+0000","last_change":"2023-10-09T04:34:09.901145+0000","last_active":"2023-10-09T04:37:28.952442+0000","last_peered":"2023-10-09T04:37:28.952442+0000","last_clean":"2023-10-09T04:37:28.952442+0000","last_became_active":"2023-10-09T04:34:09.900787+0000","last_became_peered":"2023-10-09T04:34:09.900787+0000","last_unstale":"2023-10-09T04:37:28.952442+0000","last_undegraded":"2023-10-09T04:37:28.952442+0000","last_fullsized":"2023-10-09T04:37:28.952442+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":157,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.955179+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:37:27.955179+0000","last_peered":"2023-10-09T04:37:27.955179+0000","last_clean":"2023-10-09T04:37:27.955179+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:37:27.955179+0000","last_undegraded":"2023-10-09T04:37:27.955179+0000","last_fullsized":"2023-10-09T04:37:27.955179+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":150,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.736019+0000","last_change":"2023-10-09T04:34:36.645881+0000","last_active":"2023-10-09T04:37:27.736019+0000","last_peered":"2023-10-09T04:37:27.736019+0000","last_clean":"2023-10-09T04:37:27.736019+0000","last_became_active":"2023-10-09T04:34:36.644899+0000","last_became_peered":"2023-10-09T04:34:36.644899+0000","last_unstale":"2023-10-09T04:37:27.736019+0000","last_undegraded":"2023-10-09T04:37:27.736019+0000","last_fullsized":"2023-10-09T04:37:27.736019+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:11.020741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-10-09T04:37:27.737451+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:37:27.737451+0000","last_peered":"2023-10-09T04:37:27.737451+0000","last_clean":"2023-10-09T04:37:27.737451+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:37:27.737451+0000","last_undegraded":"2023-10-09T04:37:27.737451+0000","last_fullsized":"2023-10-09T04:37:27.737451+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2182086656,"num_objects":33296,"num_object_clones":0,"num_object_copies":99888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":33296,"num_write_kb":2130944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6712918016,"data_stored":6712918016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":33296,"ondisk_log_size":33296,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41128,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430562,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001984,"kb_used_data":704760,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92743168,"statfs":{"total":95995035648,"available":94969004032,"internally_reserved":0,"allocated":721674240,"data_stored":721131181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:36:59 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.099,"15min":1.8700000000000001},"min":{"1min":0.28199999999999997,"5min":0.255,"15min":0.255},"max":{"1min":2.7320000000000002,"5min":5.5949999999999998,"15min":250.65700000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.502,"5min":1.6850000000000001,"15min":2.319},"min":{"1min":0.56299999999999994,"5min":0.27300000000000002,"15min":0.23400000000000001},"max":{"1min":8.0299999999999994,"5min":11.608000000000001,"15min":251.214},"last":1.264}]},{"osd":1,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.1719999999999999,"15min":1.0680000000000001},"min":{"1min":0.441,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.1160000000000001,"5min":5.9290000000000003,"15min":7.282},"last":5.7649999999999997},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.679,"15min":1.4059999999999999},"min":{"1min":0.48099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.879,"5min":11.555999999999999,"15min":11.988},"last":2.3290000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.609,"15min":1.6879999999999999},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.5120000000000005,"5min":8.5120000000000005,"15min":106.456},"last":3.8599999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.361,"15min":1.6759999999999999},"min":{"1min":0.64400000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.8010000000000002,"5min":7.4379999999999997,"15min":106.708},"last":0.41199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.7989999999999999,"15min":1.3109999999999999},"min":{"1min":0.52600000000000002,"5min":0.39200000000000002,"15min":0.318},"max":{"1min":1.1659999999999999,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":2.1120000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.595,"15min":1.2569999999999999},"min":{"1min":0.60899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.9370000000000001,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.345,"15min":1.2669999999999999},"min":{"1min":0.625,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1899999999999999,"5min":11.683999999999999,"15min":11.683999999999999},"last":1.135},{"interface":"front","average":{"1min":0.876,"5min":1.341,"15min":1.1659999999999999},"min":{"1min":0.61299999999999999,"5min":0.317,"15min":0.317},"max":{"1min":1.1950000000000001,"5min":10.696999999999999,"15min":10.696999999999999},"last":1.7509999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.8360000000000001,"15min":1.9630000000000001},"min":{"1min":0.56899999999999995,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.278,"5min":8.0790000000000006,"15min":8.0790000000000006},"last":1.167},{"interface":"front","average":{"1min":1.0029999999999999,"5min":2.052,"15min":2.2879999999999998},"min":{"1min":0.54900000000000004,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.395,"5min":10.786,"15min":10.786},"last":2.1509999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.768,"15min":1.7410000000000001},"min":{"1min":0.71199999999999997,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.3259999999999996,"5min":11.955,"15min":11.955},"last":1.1910000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.1819999999999999,"15min":1.181},"min":{"1min":0.47599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.367,"5min":4.351,"15min":4.351},"last":0.438}]}]},{"osd":6,"up_from":146,"seq":627065225262,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967288,"kb_used_data":629040,"kb_used_omap":5,"kb_used_meta":338234,"kb_avail":92777864,"statfs":{"total":95995035648,"available":95004532736,"internally_reserved":0,"allocated":644136960,"data_stored":643601966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":346352224},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":3.1869999999999998,"15min":2.5910000000000002},"min":{"1min":0.42899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.1699999999999999,"5min":125.666,"15min":125.666},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.75800000000000001,"5min":3.1880000000000002,"15min":2.5819999999999999},"min":{"1min":0.46100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.3979999999999999,"5min":124.991,"15min":124.991},"last":1.542}]},{"osd":1,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":3.423,"15min":4.25},"min":{"1min":0.48399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.284,"5min":68.155000000000001,"15min":68.155000000000001},"last":1.5089999999999999},{"interface":"front","average":{"1min":0.84299999999999997,"5min":3.9889999999999999,"15min":4.9649999999999999},"min":{"1min":0.55100000000000005,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.4169999999999998,"5min":67.308999999999997,"15min":67.308999999999997},"last":0.52900000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":3.9449999999999998,"15min":5.0030000000000001},"min":{"1min":0.51200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.02,"5min":68.316000000000003,"15min":68.316000000000003},"last":1.569},{"interface":"front","average":{"1min":0.98299999999999998,"5min":3.9750000000000001,"15min":4.8499999999999996},"min":{"1min":0.80300000000000005,"5min":0.313,"15min":0.313},"max":{"1min":1.228,"5min":68.210999999999999,"15min":68.210999999999999},"last":0.72499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":2.125,"15min":2.1059999999999999},"min":{"1min":0.64800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.3280000000000001,"5min":11.673,"15min":11.673},"last":0.629},{"interface":"front","average":{"1min":0.90300000000000002,"5min":2.3439999999999999,"15min":2.452},"min":{"1min":0.56699999999999995,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.5150000000000001,"5min":15.824,"15min":15.824},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5049999999999999,"15min":1.629},"min":{"1min":0.59199999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.4870000000000001,"5min":8.0500000000000007,"15min":8.0500000000000007},"last":1.891},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.5389999999999999,"15min":1.4990000000000001},"min":{"1min":0.628,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1579999999999999,"5min":10.324,"15min":10.324},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:37:43 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.073,"15min":2.4830000000000001},"min":{"1min":0.64100000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.113,"5min":10.193,"15min":10.193},"last":1.798},{"interface":"front","average":{"1min":1.129,"5min":2.1080000000000001,"15min":2.3599999999999999},"min":{"1min":0.65300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3660000000000001,"5min":11.727,"15min":11.727},"last":0.69699999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.9339999999999999,"15min":2.073},"min":{"1min":0.65700000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":2.5550000000000002,"5min":13.191000000000001,"15min":13.191000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.661,"15min":1.75},"min":{"1min":0.78900000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.3180000000000001,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":1.8520000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958711,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051820,"kb_used_data":804836,"kb_used_omap":6,"kb_used_meta":246969,"kb_avail":92693332,"statfs":{"total":95995035648,"available":94917971968,"internally_reserved":0,"allocated":824152064,"data_stored":823396124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":252896268},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.4330000000000001,"15min":1.9359999999999999},"min":{"1min":0.30199999999999999,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":5.9320000000000004,"5min":13.220000000000001,"15min":158.435},"last":6.3310000000000004},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4059999999999999,"15min":2.052},"min":{"1min":0.373,"5min":0.218,"15min":0.218},"max":{"1min":6.9089999999999998,"5min":7.1210000000000004,"15min":159.38999999999999},"last":6.2169999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.2589999999999999,"15min":1.881},"min":{"1min":0.442,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":2.4630000000000001,"5min":9.4589999999999996,"15min":157.70400000000001},"last":2.4630000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.248,"15min":1.9299999999999999},"min":{"1min":0.47799999999999998,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":3.0270000000000001,"5min":9.1780000000000008,"15min":157.64099999999999},"last":2.9369999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:37:26 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.8799999999999999,"15min":1.409},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.27400000000000002},"max":{"1min":1.135,"5min":14.369,"15min":14.369},"last":2.6600000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.9019999999999999,"15min":1.345},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.249},"max":{"1min":0.92200000000000004,"5min":12.932,"15min":12.932},"last":2.698}]},{"osd":4,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.3089999999999999,"15min":1.2529999999999999},"min":{"1min":0.5,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":2.4950000000000001,"5min":12.327,"15min":12.327},"last":3.0569999999999999},{"interface":"front","average":{"1min":1.1990000000000001,"5min":1.256,"15min":1.3200000000000001},"min":{"1min":0.48399999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.0629999999999997,"5min":12.209,"15min":12.209},"last":2.5950000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.579,"15min":1.7949999999999999},"min":{"1min":0.48399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.0800000000000001,"5min":6.9900000000000002,"15min":6.9900000000000002},"last":2.7210000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6379999999999999,"15min":1.8939999999999999},"min":{"1min":0.49099999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.649,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":2.8849999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.768,"15min":2.0190000000000001},"min":{"1min":0.58699999999999997,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":7.0220000000000002,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":2.7370000000000001},{"interface":"front","average":{"1min":1.248,"5min":1.8140000000000001,"15min":2.0579999999999998},"min":{"1min":0.64500000000000002,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":7.048,"5min":8.9879999999999995,"15min":8.9879999999999995},"last":2.976}]},{"osd":7,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.7480000000000002,"15min":1.734},"min":{"1min":0.44600000000000001,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":1.323,"5min":144.613,"15min":144.613},"last":2.923},{"interface":"front","average":{"1min":1.2589999999999999,"5min":2.5329999999999999,"15min":1.7410000000000001},"min":{"1min":0.56100000000000005,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.1050000000000004,"5min":145.53700000000001,"15min":145.53700000000001},"last":2.5379999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122236,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1227828,"kb_used_data":943788,"kb_used_omap":9,"kb_used_meta":284022,"kb_avail":92517324,"statfs":{"total":95995035648,"available":94737739776,"internally_reserved":0,"allocated":966438912,"data_stored":965677543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":290839025},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":1.982,"15min":1.3999999999999999},"min":{"1min":0.37,"5min":0.223,"15min":0.18099999999999999},"max":{"1min":0.81799999999999995,"5min":61.372999999999998,"15min":61.372999999999998},"last":2.161},{"interface":"front","average":{"1min":0.62,"5min":1.5409999999999999,"15min":1.341},"min":{"1min":0.23000000000000001,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":1.038,"5min":25.931000000000001,"15min":43.091999999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.992,"15min":1.5940000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.6059999999999999,"5min":61.634999999999998,"15min":61.634999999999998},"last":6.218},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.829,"15min":1.405},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":2.742,"5min":64.677000000000007,"15min":64.677000000000007},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":2.0649999999999999,"15min":1.2749999999999999},"min":{"1min":0.44,"5min":0.27400000000000002,"15min":0.20300000000000001},"max":{"1min":1.2909999999999999,"5min":64.622,"15min":64.622},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":2.129,"15min":1.385},"min":{"1min":0.38700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":1.1140000000000001,"5min":64.540999999999997,"15min":64.540999999999997},"last":0.70199999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.909,"15min":1.5269999999999999},"min":{"1min":0.34200000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.9960000000000004,"5min":61.549999999999997,"15min":61.549999999999997},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.7529999999999999,"15min":1.3109999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.308},"max":{"1min":1.702,"5min":61.902999999999999,"15min":61.902999999999999},"last":0.90300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.8809999999999998,"15min":3.633},"min":{"1min":0.47199999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.4850000000000001,"5min":61.972999999999999,"15min":61.972999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":3.419,"15min":4.2930000000000001},"min":{"1min":0.56799999999999995,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.425,"5min":62.417000000000002,"15min":62.417000000000002},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:37:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":3.3799999999999999,"15min":3.9049999999999998},"min":{"1min":0.52700000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.536,"5min":62.462000000000003,"15min":62.462000000000003},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.91600000000000004,"5min":3.5379999999999998,"15min":4.1760000000000002},"min":{"1min":0.41799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.851,"5min":61.808,"15min":61.808},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.056,"15min":1.6910000000000001},"min":{"1min":0.504,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6189999999999998,"5min":62.006,"15min":62.006},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.347,"5min":2.1699999999999999,"15min":1.734},"min":{"1min":0.47199999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0359999999999996,"5min":61.606999999999999,"15min":61.606999999999999},"last":0.63}]}]},{"osd":2,"up_from":98,"seq":420906795185,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094184,"kb_used_data":774560,"kb_used_omap":11,"kb_used_meta":319604,"kb_avail":92650968,"statfs":{"total":95995035648,"available":94874591232,"internally_reserved":0,"allocated":793149440,"data_stored":792397157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":327274538},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.512,"15min":1.1830000000000001},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.191},"max":{"1min":7.2789999999999999,"5min":18.155999999999999,"15min":18.155999999999999},"last":2.7250000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.518,"15min":1.228},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20599999999999999},"max":{"1min":1.6539999999999999,"5min":14.548,"15min":14.548},"last":0.436}]},{"osd":1,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.25},"max":{"1min":3.024,"5min":14.438000000000001,"15min":14.438000000000001},"last":3.024},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.3520000000000001,"15min":1.252},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.1859999999999999,"5min":13.728,"15min":13.728},"last":0.93799999999999994}]},{"osd":3,"last update":"Mon Oct 9 04:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.796,"15min":1.2609999999999999},"min":{"1min":0.41099999999999998,"5min":0.29599999999999999,"15min":0.27800000000000002},"max":{"1min":4.5279999999999996,"5min":14.515000000000001,"15min":14.515000000000001},"last":2.7890000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.782,"15min":1.349},"min":{"1min":0.41199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.4180000000000001,"5min":14.66,"15min":14.66},"last":0.77500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.371,"15min":1.339},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.29599999999999999},"max":{"1min":1.272,"5min":13.689,"15min":13.689},"last":0.80600000000000005},{"interface":"front","average":{"1min":2.028,"5min":1.704,"15min":1.5529999999999999},"min":{"1min":0.625,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":10.002000000000001,"5min":13.759,"15min":13.759},"last":5.7089999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.5899999999999999,"15min":3.0600000000000001},"min":{"1min":0.46200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.057,"5min":14.564,"15min":14.564},"last":5.766},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.8029999999999999,"15min":2.0169999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.272,"5min":14.305999999999999,"15min":14.305999999999999},"last":2.8559999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:37:16 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6379999999999999,"15min":1.6279999999999999},"min":{"1min":0.46000000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2570000000000001,"5min":9.5719999999999992,"15min":9.5719999999999992},"last":2.8260000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.548,"15min":1.4159999999999999},"min":{"1min":0.69299999999999995,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.4430000000000001,"5min":14.683999999999999,"15min":14.683999999999999},"last":0.85499999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.532,"15min":1.3839999999999999},"min":{"1min":0.54000000000000004,"5min":0.36199999999999999,"15min":0.35099999999999998},"max":{"1min":2.8820000000000001,"5min":14.372999999999999,"15min":14.372999999999999},"last":2.8820000000000001},{"interface":"front","average":{"1min":1.111,"5min":1.577,"15min":1.425},"min":{"1min":0.56699999999999995,"5min":0.38400000000000001,"15min":0.29899999999999999},"max":{"1min":2.3820000000000001,"5min":14.6,"15min":14.6},"last":0.82899999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631657,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533740,"kb_used_data":1224868,"kb_used_omap":5,"kb_used_meta":308858,"kb_avail":92211412,"statfs":{"total":95995035648,"available":94424485888,"internally_reserved":0,"allocated":1254264832,"data_stored":1253508795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":316270766},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.2150000000000001,"15min":1.409},"min":{"1min":0.249,"5min":0.154,"15min":0.154},"max":{"1min":0.79400000000000004,"5min":9.6379999999999999,"15min":61.966999999999999},"last":1.3979999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.3759999999999999,"15min":1.4630000000000001},"min":{"1min":0.36399999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.88100000000000001,"5min":12.638999999999999,"15min":62.052999999999997},"last":1.9239999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.2170000000000001,"15min":1.238},"min":{"1min":0.317,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.77900000000000003,"5min":11.920999999999999,"15min":61.395000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.1539999999999999,"15min":1.2150000000000001},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.19500000000000001},"max":{"1min":0.94699999999999995,"5min":5.5839999999999996,"15min":61.179000000000002},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.6899999999999999,"15min":2.0649999999999999},"min":{"1min":0.38900000000000001,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":0.99399999999999999,"5min":12.087999999999999,"15min":181.959},"last":11.297000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1.4790000000000001,"15min":1.9930000000000001},"min":{"1min":0.34399999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.002,"5min":6.9130000000000003,"15min":181.91200000000001},"last":10.663}]},{"osd":4,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.1930000000000001,"15min":1.0649999999999999},"min":{"1min":0.44600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":1.0649999999999999,"5min":7.3520000000000003,"15min":7.3520000000000003},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3089999999999999,"15min":1.155},"min":{"1min":0.496,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.2509999999999999,"5min":7.9260000000000002,"15min":7.9260000000000002},"last":1.8460000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.542,"15min":1.7749999999999999},"min":{"1min":0.52500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1679999999999999,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":1.879},{"interface":"front","average":{"1min":0.88500000000000001,"5min":2.0590000000000002,"15min":2.4729999999999999},"min":{"1min":0.48899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.325,"5min":7.976,"15min":7.976},"last":1.901}]},{"osd":6,"last update":"Mon Oct 9 04:37:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":2.2869999999999999,"15min":2.4289999999999998},"min":{"1min":0.56999999999999995,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.22,"5min":11.842000000000001,"15min":11.842000000000001},"last":2.1200000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.7330000000000001,"15min":1.7949999999999999},"min":{"1min":0.63700000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2270000000000001,"5min":8.5020000000000007,"15min":8.5020000000000007},"last":0.53400000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":4.3849999999999998,"15min":2.2029999999999998},"min":{"1min":0.68000000000000005,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.292,"5min":198.012,"15min":198.012},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.98299999999999998,"5min":4.0629999999999997,"15min":2.117},"min":{"1min":0.61899999999999999,"5min":0.47399999999999998,"15min":0.34599999999999997},"max":{"1min":1.3280000000000001,"5min":198.05199999999999,"15min":198.05199999999999},"last":0.57399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019259,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032824,"kb_used_data":723312,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92712328,"statfs":{"total":95995035648,"available":94937423872,"internally_reserved":0,"allocated":740671488,"data_stored":740136007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1679999999999999,"15min":1.379},"min":{"1min":0.50700000000000001,"5min":0.26100000000000001,"15min":0.24099999999999999},"max":{"1min":4.9080000000000004,"5min":12.295999999999999,"15min":48.450000000000003},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.665,"5min":1.502,"15min":1.581},"min":{"1min":0.56699999999999995,"5min":0.31,"15min":0.28599999999999998},"max":{"1min":8.5990000000000002,"5min":9.1509999999999998,"15min":49.283000000000001},"last":3.2909999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.02,"15min":1.5},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.27300000000000002},"max":{"1min":1.3540000000000001,"5min":79.361999999999995,"15min":79.361999999999995},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.79800000000000004,"5min":2.073,"15min":1.613},"min":{"1min":0.39500000000000002,"5min":0.30399999999999999,"15min":0.29199999999999998},"max":{"1min":1.425,"5min":79.292000000000002,"15min":79.292000000000002},"last":1.1699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.2480000000000002,"15min":1.754},"min":{"1min":0.70899999999999996,"5min":0.31900000000000001,"15min":0.31},"max":{"1min":2.6930000000000001,"5min":77.537000000000006,"15min":77.537000000000006},"last":1.6020000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.907,"15min":1.796},"min":{"1min":0.56200000000000006,"5min":0.36099999999999999,"15min":0.311},"max":{"1min":1.2390000000000001,"5min":77.188000000000002,"15min":77.188000000000002},"last":3.8140000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.532,"15min":1.2210000000000001},"min":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":1.456,"5min":10.417,"15min":10.417},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.54,"15min":1.319},"min":{"1min":0.53300000000000003,"5min":0.34599999999999997,"15min":0.32000000000000001},"max":{"1min":1.391,"5min":8.7899999999999991,"15min":11.112},"last":1.0329999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.1479999999999999,"15min":1.105},"min":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2549999999999999,"5min":10.412000000000001,"15min":10.412000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.6180000000000001,"15min":1.712},"min":{"1min":0.77200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":2.6480000000000001,"5min":6.4470000000000001,"15min":6.4470000000000001},"last":0.91700000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.997,"15min":0.90300000000000002},"min":{"1min":0.52000000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.5669999999999999,"5min":3.802,"15min":3.802},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.999,"5min":1.3220000000000001,"15min":1.2809999999999999},"min":{"1min":0.60799999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.27,"5min":10.257999999999999,"15min":10.257999999999999},"last":0.98399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.0629999999999999,"15min":1.2370000000000001},"min":{"1min":0.72299999999999998,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.5289999999999999,"5min":8.734,"15min":11.725},"last":1.526},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.365,"15min":1.242},"min":{"1min":0.56499999999999995,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.6359999999999999,"5min":13.349,"15min":13.349},"last":1.5600000000000001}]}]},{"osd":5,"up_from":150,"seq":644245094441,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986216,"kb_used_data":640608,"kb_used_omap":6,"kb_used_meta":345593,"kb_avail":92758936,"statfs":{"total":95995035648,"available":94985150464,"internally_reserved":0,"allocated":655982592,"data_stored":655458943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":353887507},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.7050000000000001,"15min":1.7929999999999999},"min":{"1min":0.53900000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.0339999999999998,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":1.873,"15min":2.0390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.389,"5min":8.2319999999999993,"15min":8.2319999999999993},"last":1}]},{"osd":1,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.9039999999999999,"15min":2.169},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.2530000000000001,"5min":7.2750000000000004,"15min":7.2750000000000004},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7030000000000001,"15min":1.8129999999999999},"min":{"1min":0.58999999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.1930000000000001,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":1.577}]},{"osd":2,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":2.1499999999999999,"15min":2.4020000000000001},"min":{"1min":0.49299999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.6200000000000001,"5min":12.69,"15min":12.69},"last":1.448},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.0329999999999999,"15min":2.1989999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.28,"5min":10.657999999999999,"15min":10.657999999999999},"last":1.0640000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.694,"15min":1.8799999999999999},"min":{"1min":0.46400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3240000000000001,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":2.25,"15min":2.6600000000000001},"min":{"1min":0.60599999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.3520000000000001,"5min":8.2859999999999996,"15min":8.2859999999999996},"last":0.93300000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.738,"15min":1.972},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.4319999999999999,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.4330000000000001,"15min":1.5720000000000001},"min":{"1min":0.51700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.4119999999999999,"5min":6.8300000000000001,"15min":6.8300000000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.649,"15min":1.833},"min":{"1min":0.71199999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.569,"5min":11.087,"15min":11.087},"last":0.502},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.8140000000000001,"15min":2.0600000000000001},"min":{"1min":0.47299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":2.7869999999999999,"5min":9.3659999999999997,"15min":9.3659999999999997},"last":0.83299999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":2.3290000000000002,"15min":2.3340000000000001},"min":{"1min":0.48299999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.6759999999999999,"5min":39.771000000000001,"15min":39.771000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.0640000000000001,"5min":2.746,"15min":3.0579999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.506,"5min":39.905000000000001,"15min":39.905000000000001},"last":0.71099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1001848832,"data_stored":1001848832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":833159168,"data_stored":833159168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":828309504,"data_stored":828309504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1298268160,"data_stored":1298268160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":751370240,"data_stored":751370240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":594935808,"data_stored":594935808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":665583616,"data_stored":665583616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":739442688,"data_stored":739442688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:38:01.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:38:01.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:01.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:01.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:01.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:01.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:01.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:01 smithi082 bash[79733]: cluster 2023-10-09T04:38:00.439069+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:38:01.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:01 smithi082 bash[77063]: cluster 2023-10-09T04:38:00.439069+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:38:01.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:01.997 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:01 smithi153 bash[59565]: cluster 2023-10-09T04:38:00.439069+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T04:38:02.023 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:02.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:02.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 45079 45063 108.296 82.9375 0.00728266 0.00922766 2023-10-09T04:38:02.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:02.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:02.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:02.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:02.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:02.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:02.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:02.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:02.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:02.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:02.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:02.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:03.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:03.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:03.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 46433 46417 107.42 84.625 0.0164434 0.00930203 2023-10-09T04:38:03.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:03.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:03.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:03.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:03.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:03.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:03.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:03.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:38:03.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.749 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:03.852 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:03.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:03.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:03 smithi082 bash[77063]: cluster 2023-10-09T04:38:02.441200+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:38:03.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:03 smithi082 bash[79733]: cluster 2023-10-09T04:38:02.441200+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:38:03.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:04.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:04.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:04.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:03 smithi153 bash[59565]: cluster 2023-10-09T04:38:02.441200+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:38:04.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:04.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 16 47713 47697 106.44 80 0.0117699 0.00939014 2023-10-09T04:38:04.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:04.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:04.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:04.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:04.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:04.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:04.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:04.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:04.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:04.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:04.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:04.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:04.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:05.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:05.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:05.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 49097 49081 105.752 86.5 0.0144665 0.00945045 2023-10-09T04:38:05.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:05.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:05.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:05.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:05.425 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:38:05.425 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:38:05.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:05.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:05.543 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:05.543 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:05.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:05.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:05.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:05.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:05.782 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:05 smithi082 bash[77063]: cluster 2023-10-09T04:38:04.442014+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:38:05.782 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:05 smithi082 bash[79733]: cluster 2023-10-09T04:38:04.442014+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:38:05.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:05.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:05.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:05.915 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:05 smithi153 bash[59565]: cluster 2023-10-09T04:38:04.442014+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:38:05.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:05.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:06.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:06.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:06.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 50209 50193 104.544 69.5 0.00758756 0.00955898 2023-10-09T04:38:06.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:06.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:06.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:06.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:06.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:06.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:06.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:06.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:06.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:06.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:06.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:06.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:06.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:06.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:06.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:07.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:07.169 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 51491 51475 103.756 80.125 0.00964057 0.00963234 2023-10-09T04:38:07.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:07.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:07.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:07.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:07.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:07.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:07.779 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:07 smithi082 bash[77063]: cluster 2023-10-09T04:38:06.443033+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:07.780 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:07 smithi082 bash[79733]: cluster 2023-10-09T04:38:06.443033+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:07.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:07.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:07.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:07.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:08.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:07 smithi153 bash[59565]: cluster 2023-10-09T04:38:06.443033+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:08.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:08.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 52791 52775 103.053 81.25 0.00875006 0.00969747 2023-10-09T04:38:08.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:08.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:08.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:08.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:08.544 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:08.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:08.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:08.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:08.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:08.829 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:08.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:08.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:08.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:09.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:09.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:09.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 54028 54012 102.273 77.3125 0.00561235 0.00977127 2023-10-09T04:38:09.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:09.325 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:09.367 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:09.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:38:09.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:09.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:09.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:09.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:09.785 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:09 smithi082 bash[77063]: cluster 2023-10-09T04:38:08.443615+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:38:09.785 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:09 smithi082 bash[79733]: cluster 2023-10-09T04:38:08.443615+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:38:09.786 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:09] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.33.4" 2023-10-09T04:38:09.819 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:09.861 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:09.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:09.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:09.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:10.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:10.065 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:09 smithi153 bash[59565]: cluster 2023-10-09T04:38:08.443615+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:38:10.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:10.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 55139 55123 101.307 69.4375 0.00419613 0.00986508 2023-10-09T04:38:10.203 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:10.224 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:10.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:10.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:10.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:10.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:10.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:10.723 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:10.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.881 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:10.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:10.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:11.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:11.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:11.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:11.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 56396 56380 100.657 78.5625 0.00895499 0.00992906 2023-10-09T04:38:11.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:11.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:11.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:11.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:11.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:11.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:11.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:11.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:11.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:11.780 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:11 smithi153 bash[59565]: cluster 2023-10-09T04:38:10.444196+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:38:11.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:11.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:11.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:11 smithi082 bash[77063]: cluster 2023-10-09T04:38:10.444196+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:38:11.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:11 smithi082 bash[79733]: cluster 2023-10-09T04:38:10.444196+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:38:11.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:11.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:12.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:12.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:12.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 57633 57617 100.008 77.3125 0.0161543 0.00999321 2023-10-09T04:38:12.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:12.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:12.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:12.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.581 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:12.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.719 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:12.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:12.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:12.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:12.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.986 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:12.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:13.093 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:13.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:13.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 58776 58760 99.2359 71.4375 0.0129116 0.0100722 2023-10-09T04:38:13.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:13.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:13.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:13.383 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:13.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:13.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:13.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:13.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:13.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:13.764 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:13 smithi082 bash[77063]: cluster 2023-10-09T04:38:12.444650+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:38:13.764 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:13 smithi082 bash[79733]: cluster 2023-10-09T04:38:12.444650+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:38:13.779 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:13.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:13.890 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:13 smithi153 bash[59565]: cluster 2023-10-09T04:38:12.444650+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:38:13.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:13.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:13.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:14.014 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:14.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:14.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 59844 59828 98.381 66.75 0.022286 0.0101581 2023-10-09T04:38:14.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:14.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:14.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:14.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:14.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:14.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:14.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:14.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:14.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:14.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.837 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:14.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:14.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:14 smithi153 bash[59565]: audit 2023-10-09T04:38:14.273483+0000 mon.a (mon.0) 1061 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:14.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:14.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:14.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:14 smithi082 bash[77063]: audit 2023-10-09T04:38:14.273483+0000 mon.a (mon.0) 1061 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:14 smithi082 bash[79733]: audit 2023-10-09T04:38:14.273483+0000 mon.a (mon.0) 1061 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:15.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:15.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:15.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 60921 60905 97.5843 67.3125 0.00853082 0.0102418 2023-10-09T04:38:15.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:15.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:15.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:15.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:15.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:15.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:15.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:15.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:15.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:15.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:15.762 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:15.771 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:15 smithi153 bash[59565]: cluster 2023-10-09T04:38:14.446958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:38:15.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:15.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:15.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:15.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:15.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:15 smithi082 bash[77063]: cluster 2023-10-09T04:38:14.446958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:38:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:15 smithi082 bash[79733]: cluster 2023-10-09T04:38:14.446958+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:38:16.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:16.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:16.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:16.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:38:16.169808+0000 min lat: 0.00131815 max lat: 0.275588 avg lat: 0.0103043 2023-10-09T04:38:16.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:16.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 62103 62087 96.9914 73.875 0.0153248 0.0103043 2023-10-09T04:38:16.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:16.294 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:16.334 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:16.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:16.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:16.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:16.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:16.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:16.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:16.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:16.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:16.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:16.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:16.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:16.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:17.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:17.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:17.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:17.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 63292 63276 96.4382 74.3125 0.00921221 0.0103644 2023-10-09T04:38:17.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:17.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:17.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:17.411 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:17.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:17.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:17.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:17.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:17.724 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:17 smithi082 bash[77063]: cluster 2023-10-09T04:38:16.447519+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:38:17.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:17 smithi082 bash[79733]: cluster 2023-10-09T04:38:16.447519+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:38:17.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:17.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:17 smithi153 bash[59565]: cluster 2023-10-09T04:38:16.447519+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:38:17.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:17.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:18.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:18.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:18.170 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 64387 64371 95.7714 68.4375 0.0147999 0.010436 2023-10-09T04:38:18.183 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:18.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:18.347 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:18.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:18.583 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:18.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:18.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:18.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:18.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:18.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:18.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:18.914 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:18.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:19.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:19.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:19.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:19.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 65501 65485 95.1633 69.625 0.0201609 0.010502 2023-10-09T04:38:19.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:19.273 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:19.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:19.416 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:19.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:19.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:19.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:19.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.797 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:19 smithi153 bash[59565]: cluster 2023-10-09T04:38:18.448075+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:38:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:19.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:19.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:19 smithi082 bash[77063]: cluster 2023-10-09T04:38:18.448075+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:38:19.910 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:19] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.33.4" 2023-10-09T04:38:19.910 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:19 smithi082 bash[79733]: cluster 2023-10-09T04:38:18.448075+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:38:19.929 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:19.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:20.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:20.063 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:20.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:20.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:20.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 66434 66418 94.3257 58.3125 0.00553496 0.0105956 2023-10-09T04:38:20.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:20.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:20.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:20.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:20.581 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:20.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:20.713 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:20.761 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:38:20.766 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:38:20.772 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:38:20.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.881 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:20.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:20.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:20.996 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:20 smithi153 bash[59565]: audit 2023-10-09T04:38:19.963412+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:21.013 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:21.043 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:20 smithi082 bash[77063]: audit 2023-10-09T04:38:19.963412+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:21.044 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:20 smithi082 bash[79733]: audit 2023-10-09T04:38:19.963412+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:21.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:21.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 67418 67402 93.5962 61.5 0.00778097 0.0106784 2023-10-09T04:38:21.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:21.283 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:21.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:21.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:21.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:21.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:21.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:21.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: cluster 2023-10-09T04:38:20.451175+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:21.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: audit 2023-10-09T04:38:20.726980+0000 mgr.y (mgr.34107) 725 : audit 0 from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: cluster 2023-10-09T04:38:20.730992+0000 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: cluster 2023-10-09T04:38:20.745575+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: cluster 2023-10-09T04:38:20.748116+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: audit 2023-10-09T04:38:20.753973+0000 mon.a (mon.0) 1064 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:21.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:21 smithi153 bash[59565]: cluster 2023-10-09T04:38:20.754395+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:38:21.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:21.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: cluster 2023-10-09T04:38:20.451175+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: audit 2023-10-09T04:38:20.726980+0000 mgr.y (mgr.34107) 725 : audit 0 from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: cluster 2023-10-09T04:38:20.730992+0000 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: cluster 2023-10-09T04:38:20.745575+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: cluster 2023-10-09T04:38:20.748116+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: audit 2023-10-09T04:38:20.753973+0000 mon.a (mon.0) 1064 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[77063]: cluster 2023-10-09T04:38:20.754395+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: cluster 2023-10-09T04:38:20.451175+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: audit 2023-10-09T04:38:20.726980+0000 mgr.y (mgr.34107) 725 : audit 0 from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: cluster 2023-10-09T04:38:20.730992+0000 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: cluster 2023-10-09T04:38:20.745575+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: cluster 2023-10-09T04:38:20.748116+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T04:38:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: audit 2023-10-09T04:38:20.753973+0000 mon.a (mon.0) 1064 : audit 1 from='client.? 172.21.15.82:0/3604685804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:21.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:21 smithi082 bash[79733]: cluster 2023-10-09T04:38:20.754395+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:38:22.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:22.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:22.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:22.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:22.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 68430 68414 92.9364 63.25 0.131104 0.0107335 2023-10-09T04:38:22.202 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:22.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:22.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:22.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:22.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:22.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:22.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:22.697 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:22.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:22.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:22.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:22.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:22.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:22.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:23.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:23.173 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 16 69513 69497 92.3991 67.6875 0.00325457 0.0108165 2023-10-09T04:38:23.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:23.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:23.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:23.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:23.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:23.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:23.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:23.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:23.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:23.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:23.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:23.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:23.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:23.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:23.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:23 smithi082 bash[77063]: cluster 2023-10-09T04:38:22.451701+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:38:23.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:23 smithi082 bash[79733]: cluster 2023-10-09T04:38:22.451701+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:38:24.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:24.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:24.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:24.163 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:23 smithi153 bash[59565]: cluster 2023-10-09T04:38:22.451701+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:38:24.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 70393 70377 91.62 55 0.016856 0.0109091 2023-10-09T04:38:24.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:24.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:24.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:24.416 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:24.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:24.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:24.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:24.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:24.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:24.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:24.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:24.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:24.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:24.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:24.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:25.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:25.097 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:25.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 71341 71325 90.9593 59.25 0.0108095 0.0109869 2023-10-09T04:38:25.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:25.227 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:25.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:25.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:25.352 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":723,"stamp":"2023-10-09T04:38:20.450713+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4195358643,"num_objects":63971,"num_object_clones":0,"num_object_copies":191913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63971,"num_whiteouts":0,"num_read":6301,"num_read_kb":12735,"num_write":65049,"num_write_kb":4107511,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65113,"ondisk_log_size":65113,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14899740,"kb_used_data":12357660,"kb_used_omap":46,"kb_used_meta":2541905,"kb_avail":735061476,"statfs":{"total":767960285184,"available":752702951424,"internally_reserved":0,"allocated":12654243840,"data_stored":12649062688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47544,"internal_metadata":2602911304},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":859308032,"num_objects":13112,"num_object_clones":0,"num_object_copies":39336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13112,"num_write_kb":839168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007508"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":159,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761367+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:37:32.761367+0000","last_peered":"2023-10-09T04:37:32.761367+0000","last_clean":"2023-10-09T04:37:32.761367+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:37:32.761367+0000","last_undegraded":"2023-10-09T04:37:32.761367+0000","last_fullsized":"2023-10-09T04:37:32.761367+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":152,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853459+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:37:34.853459+0000","last_peered":"2023-10-09T04:37:34.853459+0000","last_clean":"2023-10-09T04:37:34.853459+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:37:34.853459+0000","last_undegraded":"2023-10-09T04:37:34.853459+0000","last_fullsized":"2023-10-09T04:37:34.853459+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853079+0000","last_change":"2023-10-09T04:36:24.634063+0000","last_active":"2023-10-09T04:37:34.853079+0000","last_peered":"2023-10-09T04:37:34.853079+0000","last_clean":"2023-10-09T04:37:34.853079+0000","last_became_active":"2023-10-09T04:36:24.633693+0000","last_became_peered":"2023-10-09T04:36:24.633693+0000","last_unstale":"2023-10-09T04:37:34.853079+0000","last_undegraded":"2023-10-09T04:37:34.853079+0000","last_fullsized":"2023-10-09T04:37:34.853079+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:34.420556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":132,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756930+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:37:32.756930+0000","last_peered":"2023-10-09T04:37:32.756930+0000","last_clean":"2023-10-09T04:37:32.756930+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:37:32.756930+0000","last_undegraded":"2023-10-09T04:37:32.756930+0000","last_fullsized":"2023-10-09T04:37:32.756930+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756125+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:37:32.756125+0000","last_peered":"2023-10-09T04:37:32.756125+0000","last_clean":"2023-10-09T04:37:32.756125+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:37:32.756125+0000","last_undegraded":"2023-10-09T04:37:32.756125+0000","last_fullsized":"2023-10-09T04:37:32.756125+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.197749+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:37:33.197749+0000","last_peered":"2023-10-09T04:37:33.197749+0000","last_clean":"2023-10-09T04:37:33.197749+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:37:33.197749+0000","last_undegraded":"2023-10-09T04:37:33.197749+0000","last_fullsized":"2023-10-09T04:37:33.197749+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.757238+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:37:32.757238+0000","last_peered":"2023-10-09T04:37:32.757238+0000","last_clean":"2023-10-09T04:37:32.757238+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:37:32.757238+0000","last_undegraded":"2023-10-09T04:37:32.757238+0000","last_fullsized":"2023-10-09T04:37:32.757238+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.663589+0000","last_change":"2023-10-09T04:34:09.918534+0000","last_active":"2023-10-09T04:37:55.663589+0000","last_peered":"2023-10-09T04:37:55.663589+0000","last_clean":"2023-10-09T04:37:55.663589+0000","last_became_active":"2023-10-09T04:34:09.918013+0000","last_became_peered":"2023-10-09T04:34:09.918013+0000","last_unstale":"2023-10-09T04:37:55.663589+0000","last_undegraded":"2023-10-09T04:37:55.663589+0000","last_fullsized":"2023-10-09T04:37:55.663589+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:00.542758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":101,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763557+0000","last_change":"2023-10-09T04:34:36.648444+0000","last_active":"2023-10-09T04:37:32.763557+0000","last_peered":"2023-10-09T04:37:32.763557+0000","last_clean":"2023-10-09T04:37:32.763557+0000","last_became_active":"2023-10-09T04:34:36.647994+0000","last_became_peered":"2023-10-09T04:34:36.647994+0000","last_unstale":"2023-10-09T04:37:32.763557+0000","last_undegraded":"2023-10-09T04:37:32.763557+0000","last_fullsized":"2023-10-09T04:37:32.763557+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":141,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852367+0000","last_change":"2023-10-09T04:34:09.912750+0000","last_active":"2023-10-09T04:37:34.852367+0000","last_peered":"2023-10-09T04:37:34.852367+0000","last_clean":"2023-10-09T04:37:34.852367+0000","last_became_active":"2023-10-09T04:34:09.908040+0000","last_became_peered":"2023-10-09T04:34:09.908040+0000","last_unstale":"2023-10-09T04:37:34.852367+0000","last_undegraded":"2023-10-09T04:37:34.852367+0000","last_fullsized":"2023-10-09T04:37:34.852367+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853505+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:37:34.853505+0000","last_peered":"2023-10-09T04:37:34.853505+0000","last_clean":"2023-10-09T04:37:34.853505+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:37:34.853505+0000","last_undegraded":"2023-10-09T04:37:34.853505+0000","last_fullsized":"2023-10-09T04:37:34.853505+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853730+0000","last_change":"2023-10-09T04:34:10.030630+0000","last_active":"2023-10-09T04:37:34.853730+0000","last_peered":"2023-10-09T04:37:34.853730+0000","last_clean":"2023-10-09T04:37:34.853730+0000","last_became_active":"2023-10-09T04:34:10.030403+0000","last_became_peered":"2023-10-09T04:34:10.030403+0000","last_unstale":"2023-10-09T04:37:34.853730+0000","last_undegraded":"2023-10-09T04:37:34.853730+0000","last_fullsized":"2023-10-09T04:37:34.853730+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:53.085670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":165,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762069+0000","last_change":"2023-10-09T04:34:09.968067+0000","last_active":"2023-10-09T04:37:32.762069+0000","last_peered":"2023-10-09T04:37:32.762069+0000","last_clean":"2023-10-09T04:37:32.762069+0000","last_became_active":"2023-10-09T04:34:09.967671+0000","last_became_peered":"2023-10-09T04:34:09.967671+0000","last_unstale":"2023-10-09T04:37:32.762069+0000","last_undegraded":"2023-10-09T04:37:32.762069+0000","last_fullsized":"2023-10-09T04:37:32.762069+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762175+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:37:32.762175+0000","last_peered":"2023-10-09T04:37:32.762175+0000","last_clean":"2023-10-09T04:37:32.762175+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:37:32.762175+0000","last_undegraded":"2023-10-09T04:37:32.762175+0000","last_fullsized":"2023-10-09T04:37:32.762175+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852133+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:37:34.852133+0000","last_peered":"2023-10-09T04:37:34.852133+0000","last_clean":"2023-10-09T04:37:34.852133+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:37:34.852133+0000","last_undegraded":"2023-10-09T04:37:34.852133+0000","last_fullsized":"2023-10-09T04:37:34.852133+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.663470+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:37:55.663470+0000","last_peered":"2023-10-09T04:37:55.663470+0000","last_clean":"2023-10-09T04:37:55.663470+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:37:55.663470+0000","last_undegraded":"2023-10-09T04:37:55.663470+0000","last_fullsized":"2023-10-09T04:37:55.663470+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756296+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:37:32.756296+0000","last_peered":"2023-10-09T04:37:32.756296+0000","last_clean":"2023-10-09T04:37:32.756296+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:37:32.756296+0000","last_undegraded":"2023-10-09T04:37:32.756296+0000","last_fullsized":"2023-10-09T04:37:32.756296+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763325+0000","last_change":"2023-10-09T04:34:36.646566+0000","last_active":"2023-10-09T04:37:32.763325+0000","last_peered":"2023-10-09T04:37:32.763325+0000","last_clean":"2023-10-09T04:37:32.763325+0000","last_became_active":"2023-10-09T04:34:36.646266+0000","last_became_peered":"2023-10-09T04:34:36.646266+0000","last_unstale":"2023-10-09T04:37:32.763325+0000","last_undegraded":"2023-10-09T04:37:32.763325+0000","last_fullsized":"2023-10-09T04:37:32.763325+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852512+0000","last_change":"2023-10-09T04:34:09.907681+0000","last_active":"2023-10-09T04:37:34.852512+0000","last_peered":"2023-10-09T04:37:34.852512+0000","last_clean":"2023-10-09T04:37:34.852512+0000","last_became_active":"2023-10-09T04:34:09.907156+0000","last_became_peered":"2023-10-09T04:34:09.907156+0000","last_unstale":"2023-10-09T04:37:34.852512+0000","last_undegraded":"2023-10-09T04:37:34.852512+0000","last_fullsized":"2023-10-09T04:37:34.852512+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756843+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:37:32.756843+0000","last_peered":"2023-10-09T04:37:32.756843+0000","last_clean":"2023-10-09T04:37:32.756843+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:37:32.756843+0000","last_undegraded":"2023-10-09T04:37:32.756843+0000","last_fullsized":"2023-10-09T04:37:32.756843+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.196865+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:37:33.196865+0000","last_peered":"2023-10-09T04:37:33.196865+0000","last_clean":"2023-10-09T04:37:33.196865+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:37:33.196865+0000","last_undegraded":"2023-10-09T04:37:33.196865+0000","last_fullsized":"2023-10-09T04:37:33.196865+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853469+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:37:34.853469+0000","last_peered":"2023-10-09T04:37:34.853469+0000","last_clean":"2023-10-09T04:37:34.853469+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:37:34.853469+0000","last_undegraded":"2023-10-09T04:37:34.853469+0000","last_fullsized":"2023-10-09T04:37:34.853469+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"178'3947","reported_seq":3961,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:18.683888+0000","last_change":"2023-10-09T04:37:13.779184+0000","last_active":"2023-10-09T04:38:18.683888+0000","last_peered":"2023-10-09T04:38:18.683888+0000","last_clean":"2023-10-09T04:38:18.683888+0000","last_became_active":"2023-10-09T04:37:13.778902+0000","last_became_peered":"2023-10-09T04:37:13.778902+0000","last_unstale":"2023-10-09T04:38:18.683888+0000","last_undegraded":"2023-10-09T04:38:18.683888+0000","last_fullsized":"2023-10-09T04:38:18.683888+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3947,"log_dups_size":0,"ondisk_log_size":3947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:15.517709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258670592,"num_objects":3947,"num_object_clones":0,"num_object_copies":11841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3947,"num_write_kb":252608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853787+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:37:34.853787+0000","last_peered":"2023-10-09T04:37:34.853787+0000","last_clean":"2023-10-09T04:37:34.853787+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:37:34.853787+0000","last_undegraded":"2023-10-09T04:37:34.853787+0000","last_fullsized":"2023-10-09T04:37:34.853787+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761298+0000","last_change":"2023-10-09T04:34:36.641326+0000","last_active":"2023-10-09T04:37:32.761298+0000","last_peered":"2023-10-09T04:37:32.761298+0000","last_clean":"2023-10-09T04:37:32.761298+0000","last_became_active":"2023-10-09T04:34:36.639824+0000","last_became_peered":"2023-10-09T04:34:36.639824+0000","last_unstale":"2023-10-09T04:37:32.761298+0000","last_undegraded":"2023-10-09T04:37:32.761298+0000","last_fullsized":"2023-10-09T04:37:32.761298+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"178'3956","reported_seq":3970,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.685061+0000","last_change":"2023-10-09T04:37:13.765303+0000","last_active":"2023-10-09T04:38:16.685061+0000","last_peered":"2023-10-09T04:38:16.685061+0000","last_clean":"2023-10-09T04:38:16.685061+0000","last_became_active":"2023-10-09T04:37:13.764983+0000","last_became_peered":"2023-10-09T04:37:13.764983+0000","last_unstale":"2023-10-09T04:38:16.685061+0000","last_undegraded":"2023-10-09T04:38:16.685061+0000","last_fullsized":"2023-10-09T04:38:16.685061+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3956,"log_dups_size":0,"ondisk_log_size":3956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.571025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259260416,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3956,"num_write_kb":253184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":172,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853629+0000","last_change":"2023-10-09T04:36:24.678230+0000","last_active":"2023-10-09T04:37:34.853629+0000","last_peered":"2023-10-09T04:37:34.853629+0000","last_clean":"2023-10-09T04:37:34.853629+0000","last_became_active":"2023-10-09T04:36:24.633399+0000","last_became_peered":"2023-10-09T04:36:24.633399+0000","last_unstale":"2023-10-09T04:37:34.853629+0000","last_undegraded":"2023-10-09T04:37:34.853629+0000","last_fullsized":"2023-10-09T04:37:34.853629+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762230+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:37:32.762230+0000","last_peered":"2023-10-09T04:37:32.762230+0000","last_clean":"2023-10-09T04:37:32.762230+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:37:32.762230+0000","last_undegraded":"2023-10-09T04:37:32.762230+0000","last_fullsized":"2023-10-09T04:37:32.762230+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"178'3949","reported_seq":3963,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.626319+0000","last_change":"2023-10-09T04:37:13.775923+0000","last_active":"2023-10-09T04:38:16.626319+0000","last_peered":"2023-10-09T04:38:16.626319+0000","last_clean":"2023-10-09T04:38:16.626319+0000","last_became_active":"2023-10-09T04:37:13.775583+0000","last_became_peered":"2023-10-09T04:37:13.775583+0000","last_unstale":"2023-10-09T04:38:16.626319+0000","last_undegraded":"2023-10-09T04:38:16.626319+0000","last_fullsized":"2023-10-09T04:38:16.626319+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3949,"log_dups_size":0,"ondisk_log_size":3949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:55.486175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801664,"num_objects":3949,"num_object_clones":0,"num_object_copies":11847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3949,"num_write_kb":252736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":156,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853473+0000","last_change":"2023-10-09T04:34:36.648398+0000","last_active":"2023-10-09T04:37:34.853473+0000","last_peered":"2023-10-09T04:37:34.853473+0000","last_clean":"2023-10-09T04:37:34.853473+0000","last_became_active":"2023-10-09T04:34:36.647896+0000","last_became_peered":"2023-10-09T04:34:36.647896+0000","last_unstale":"2023-10-09T04:37:34.853473+0000","last_undegraded":"2023-10-09T04:37:34.853473+0000","last_fullsized":"2023-10-09T04:37:34.853473+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.196991+0000","last_change":"2023-10-09T04:34:09.983232+0000","last_active":"2023-10-09T04:37:33.196991+0000","last_peered":"2023-10-09T04:37:33.196991+0000","last_clean":"2023-10-09T04:37:33.196991+0000","last_became_active":"2023-10-09T04:34:09.976353+0000","last_became_peered":"2023-10-09T04:34:09.976353+0000","last_unstale":"2023-10-09T04:37:33.196991+0000","last_undegraded":"2023-10-09T04:37:33.196991+0000","last_fullsized":"2023-10-09T04:37:33.196991+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"178'3963","reported_seq":3977,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.669385+0000","last_change":"2023-10-09T04:37:13.774933+0000","last_active":"2023-10-09T04:38:16.669385+0000","last_peered":"2023-10-09T04:38:16.669385+0000","last_clean":"2023-10-09T04:38:16.669385+0000","last_became_active":"2023-10-09T04:37:13.774485+0000","last_became_peered":"2023-10-09T04:37:13.774485+0000","last_unstale":"2023-10-09T04:38:16.669385+0000","last_undegraded":"2023-10-09T04:38:16.669385+0000","last_fullsized":"2023-10-09T04:38:16.669385+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3963,"log_dups_size":0,"ondisk_log_size":3963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.541922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259719168,"num_objects":3963,"num_object_clones":0,"num_object_copies":11889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3963,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763128+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:37:32.763128+0000","last_peered":"2023-10-09T04:37:32.763128+0000","last_clean":"2023-10-09T04:37:32.763128+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:37:32.763128+0000","last_undegraded":"2023-10-09T04:37:32.763128+0000","last_fullsized":"2023-10-09T04:37:32.763128+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.659857+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:37:55.659857+0000","last_peered":"2023-10-09T04:37:55.659857+0000","last_clean":"2023-10-09T04:37:55.659857+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:37:55.659857+0000","last_undegraded":"2023-10-09T04:37:55.659857+0000","last_fullsized":"2023-10-09T04:37:55.659857+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"178'3863","reported_seq":3875,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.655780+0000","last_change":"2023-10-09T04:37:13.769259+0000","last_active":"2023-10-09T04:38:16.655780+0000","last_peered":"2023-10-09T04:38:16.655780+0000","last_clean":"2023-10-09T04:38:16.655780+0000","last_became_active":"2023-10-09T04:37:13.768831+0000","last_became_peered":"2023-10-09T04:37:13.768831+0000","last_unstale":"2023-10-09T04:38:16.655780+0000","last_undegraded":"2023-10-09T04:38:16.655780+0000","last_fullsized":"2023-10-09T04:38:16.655780+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3863,"log_dups_size":0,"ondisk_log_size":3863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:41.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253165568,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":247232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":95,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:51.592080+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:37:51.592080+0000","last_peered":"2023-10-09T04:37:51.592080+0000","last_clean":"2023-10-09T04:37:51.592080+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:37:51.592080+0000","last_undegraded":"2023-10-09T04:37:51.592080+0000","last_fullsized":"2023-10-09T04:37:51.592080+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762096+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:37:32.762096+0000","last_peered":"2023-10-09T04:37:32.762096+0000","last_clean":"2023-10-09T04:37:32.762096+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:37:32.762096+0000","last_undegraded":"2023-10-09T04:37:32.762096+0000","last_fullsized":"2023-10-09T04:37:32.762096+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"178'3932","reported_seq":3946,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.711291+0000","last_change":"2023-10-09T04:37:13.764469+0000","last_active":"2023-10-09T04:38:16.711291+0000","last_peered":"2023-10-09T04:38:16.711291+0000","last_clean":"2023-10-09T04:38:16.711291+0000","last_became_active":"2023-10-09T04:37:13.763760+0000","last_became_peered":"2023-10-09T04:37:13.763760+0000","last_unstale":"2023-10-09T04:38:16.711291+0000","last_undegraded":"2023-10-09T04:38:16.711291+0000","last_fullsized":"2023-10-09T04:38:16.711291+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3932,"log_dups_size":0,"ondisk_log_size":3932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:08.322588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257687552,"num_objects":3932,"num_object_clones":0,"num_object_copies":11796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3932,"num_write_kb":251648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.196904+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:37:33.196904+0000","last_peered":"2023-10-09T04:37:33.196904+0000","last_clean":"2023-10-09T04:37:33.196904+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:37:33.196904+0000","last_undegraded":"2023-10-09T04:37:33.196904+0000","last_fullsized":"2023-10-09T04:37:33.196904+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762270+0000","last_change":"2023-10-09T04:36:24.632828+0000","last_active":"2023-10-09T04:37:32.762270+0000","last_peered":"2023-10-09T04:37:32.762270+0000","last_clean":"2023-10-09T04:37:32.762270+0000","last_became_active":"2023-10-09T04:36:24.632286+0000","last_became_peered":"2023-10-09T04:36:24.632286+0000","last_unstale":"2023-10-09T04:37:32.762270+0000","last_undegraded":"2023-10-09T04:37:32.762270+0000","last_fullsized":"2023-10-09T04:37:32.762270+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"178'3932","reported_seq":3944,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:18.029753+0000","last_change":"2023-10-09T04:37:13.777270+0000","last_active":"2023-10-09T04:38:18.029753+0000","last_peered":"2023-10-09T04:38:18.029753+0000","last_clean":"2023-10-09T04:38:18.029753+0000","last_became_active":"2023-10-09T04:37:13.776717+0000","last_became_peered":"2023-10-09T04:37:13.776717+0000","last_unstale":"2023-10-09T04:38:18.029753+0000","last_undegraded":"2023-10-09T04:38:18.029753+0000","last_fullsized":"2023-10-09T04:38:18.029753+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3932,"log_dups_size":0,"ondisk_log_size":3932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257687552,"num_objects":3932,"num_object_clones":0,"num_object_copies":11796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3932,"num_write_kb":251648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761799+0000","last_change":"2023-10-09T04:23:18.771465+0000","last_active":"2023-10-09T04:37:32.761799+0000","last_peered":"2023-10-09T04:37:32.761799+0000","last_clean":"2023-10-09T04:37:32.761799+0000","last_became_active":"2023-10-09T04:23:18.771119+0000","last_became_peered":"2023-10-09T04:23:18.771119+0000","last_unstale":"2023-10-09T04:37:32.761799+0000","last_undegraded":"2023-10-09T04:37:32.761799+0000","last_fullsized":"2023-10-09T04:37:32.761799+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762601+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:37:32.762601+0000","last_peered":"2023-10-09T04:37:32.762601+0000","last_clean":"2023-10-09T04:37:32.762601+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:37:32.762601+0000","last_undegraded":"2023-10-09T04:37:32.762601+0000","last_fullsized":"2023-10-09T04:37:32.762601+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"178'4067","reported_seq":4080,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:18.039088+0000","last_change":"2023-10-09T04:37:13.778215+0000","last_active":"2023-10-09T04:38:18.039088+0000","last_peered":"2023-10-09T04:38:18.039088+0000","last_clean":"2023-10-09T04:38:18.039088+0000","last_became_active":"2023-10-09T04:37:13.777797+0000","last_became_peered":"2023-10-09T04:37:13.777797+0000","last_unstale":"2023-10-09T04:38:18.039088+0000","last_undegraded":"2023-10-09T04:38:18.039088+0000","last_fullsized":"2023-10-09T04:38:18.039088+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4067,"log_dups_size":0,"ondisk_log_size":4067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266534912,"num_objects":4067,"num_object_clones":0,"num_object_copies":12201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4067,"num_write_kb":260288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":141,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852601+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:37:34.852601+0000","last_peered":"2023-10-09T04:37:34.852601+0000","last_clean":"2023-10-09T04:37:34.852601+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:37:34.852601+0000","last_undegraded":"2023-10-09T04:37:34.852601+0000","last_fullsized":"2023-10-09T04:37:34.852601+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.855039+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:37:34.855039+0000","last_peered":"2023-10-09T04:37:34.855039+0000","last_clean":"2023-10-09T04:37:34.855039+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:37:34.855039+0000","last_undegraded":"2023-10-09T04:37:34.855039+0000","last_fullsized":"2023-10-09T04:37:34.855039+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"178'4146","reported_seq":4159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:20.205719+0000","last_change":"2023-10-09T04:37:13.778496+0000","last_active":"2023-10-09T04:38:20.205719+0000","last_peered":"2023-10-09T04:38:20.205719+0000","last_clean":"2023-10-09T04:38:20.205719+0000","last_became_active":"2023-10-09T04:37:13.777643+0000","last_became_peered":"2023-10-09T04:37:13.777643+0000","last_unstale":"2023-10-09T04:38:20.205719+0000","last_undegraded":"2023-10-09T04:38:20.205719+0000","last_fullsized":"2023-10-09T04:38:20.205719+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4146,"log_dups_size":0,"ondisk_log_size":4146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271712256,"num_objects":4146,"num_object_clones":0,"num_object_copies":12438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4146,"num_write_kb":265344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":553,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.854295+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:37:34.854295+0000","last_peered":"2023-10-09T04:37:34.854295+0000","last_clean":"2023-10-09T04:37:34.854295+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:37:34.854295+0000","last_undegraded":"2023-10-09T04:37:34.854295+0000","last_fullsized":"2023-10-09T04:37:34.854295+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763499+0000","last_change":"2023-10-09T04:34:36.645924+0000","last_active":"2023-10-09T04:37:32.763499+0000","last_peered":"2023-10-09T04:37:32.763499+0000","last_clean":"2023-10-09T04:37:32.763499+0000","last_became_active":"2023-10-09T04:34:36.645572+0000","last_became_peered":"2023-10-09T04:34:36.645572+0000","last_unstale":"2023-10-09T04:37:32.763499+0000","last_undegraded":"2023-10-09T04:37:32.763499+0000","last_fullsized":"2023-10-09T04:37:32.763499+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":128,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853815+0000","last_change":"2023-10-09T04:34:36.642430+0000","last_active":"2023-10-09T04:37:34.853815+0000","last_peered":"2023-10-09T04:37:34.853815+0000","last_clean":"2023-10-09T04:37:34.853815+0000","last_became_active":"2023-10-09T04:34:36.641942+0000","last_became_peered":"2023-10-09T04:34:36.641942+0000","last_unstale":"2023-10-09T04:37:34.853815+0000","last_undegraded":"2023-10-09T04:37:34.853815+0000","last_fullsized":"2023-10-09T04:37:34.853815+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:29.110665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"178'4079","reported_seq":4092,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:18.026987+0000","last_change":"2023-10-09T04:37:13.778105+0000","last_active":"2023-10-09T04:38:18.026987+0000","last_peered":"2023-10-09T04:38:18.026987+0000","last_clean":"2023-10-09T04:38:18.026987+0000","last_became_active":"2023-10-09T04:37:13.777582+0000","last_became_peered":"2023-10-09T04:37:13.777582+0000","last_unstale":"2023-10-09T04:38:18.026987+0000","last_undegraded":"2023-10-09T04:38:18.026987+0000","last_fullsized":"2023-10-09T04:38:18.026987+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4079,"log_dups_size":0,"ondisk_log_size":4079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321344,"num_objects":4079,"num_object_clones":0,"num_object_copies":12237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4079,"num_write_kb":261056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":640,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761387+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:37:32.761387+0000","last_peered":"2023-10-09T04:37:32.761387+0000","last_clean":"2023-10-09T04:37:32.761387+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:37:32.761387+0000","last_undegraded":"2023-10-09T04:37:32.761387+0000","last_fullsized":"2023-10-09T04:37:32.761387+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761992+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:37:32.761992+0000","last_peered":"2023-10-09T04:37:32.761992+0000","last_clean":"2023-10-09T04:37:32.761992+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:37:32.761992+0000","last_undegraded":"2023-10-09T04:37:32.761992+0000","last_fullsized":"2023-10-09T04:37:32.761992+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.854161+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:37:34.854161+0000","last_peered":"2023-10-09T04:37:34.854161+0000","last_clean":"2023-10-09T04:37:34.854161+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:37:34.854161+0000","last_undegraded":"2023-10-09T04:37:34.854161+0000","last_fullsized":"2023-10-09T04:37:34.854161+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"178'3937","reported_seq":3951,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.618802+0000","last_change":"2023-10-09T04:37:13.775171+0000","last_active":"2023-10-09T04:38:16.618802+0000","last_peered":"2023-10-09T04:38:16.618802+0000","last_clean":"2023-10-09T04:38:16.618802+0000","last_became_active":"2023-10-09T04:37:13.774832+0000","last_became_peered":"2023-10-09T04:37:13.774832+0000","last_unstale":"2023-10-09T04:38:16.618802+0000","last_undegraded":"2023-10-09T04:38:16.618802+0000","last_fullsized":"2023-10-09T04:38:16.618802+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3937,"log_dups_size":0,"ondisk_log_size":3937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:58.564050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3937,"num_write_kb":251968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1289,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:53.144487+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:37:53.144487+0000","last_peered":"2023-10-09T04:37:53.144487+0000","last_clean":"2023-10-09T04:37:53.144487+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:37:53.144487+0000","last_undegraded":"2023-10-09T04:37:53.144487+0000","last_fullsized":"2023-10-09T04:37:53.144487+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852793+0000","last_change":"2023-10-09T04:34:09.901059+0000","last_active":"2023-10-09T04:37:34.852793+0000","last_peered":"2023-10-09T04:37:34.852793+0000","last_clean":"2023-10-09T04:37:34.852793+0000","last_became_active":"2023-10-09T04:34:09.900690+0000","last_became_peered":"2023-10-09T04:34:09.900690+0000","last_unstale":"2023-10-09T04:37:34.852793+0000","last_undegraded":"2023-10-09T04:37:34.852793+0000","last_fullsized":"2023-10-09T04:37:34.852793+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:59.089033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.664080+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:37:55.664080+0000","last_peered":"2023-10-09T04:37:55.664080+0000","last_clean":"2023-10-09T04:37:55.664080+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:37:55.664080+0000","last_undegraded":"2023-10-09T04:37:55.664080+0000","last_fullsized":"2023-10-09T04:37:55.664080+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"178'3873","reported_seq":3887,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.629569+0000","last_change":"2023-10-09T04:37:13.768144+0000","last_active":"2023-10-09T04:38:16.629569+0000","last_peered":"2023-10-09T04:38:16.629569+0000","last_clean":"2023-10-09T04:38:16.629569+0000","last_became_active":"2023-10-09T04:37:13.767811+0000","last_became_peered":"2023-10-09T04:37:13.767811+0000","last_unstale":"2023-10-09T04:38:16.629569+0000","last_undegraded":"2023-10-09T04:38:16.629569+0000","last_fullsized":"2023-10-09T04:38:16.629569+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3873,"log_dups_size":0,"ondisk_log_size":3873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:06.150093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820928,"num_objects":3873,"num_object_clones":0,"num_object_copies":11619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3873,"num_write_kb":247872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":622,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762983+0000","last_change":"2023-10-09T04:34:36.645853+0000","last_active":"2023-10-09T04:37:32.762983+0000","last_peered":"2023-10-09T04:37:32.762983+0000","last_clean":"2023-10-09T04:37:32.762983+0000","last_became_active":"2023-10-09T04:34:36.644515+0000","last_became_peered":"2023-10-09T04:34:36.644515+0000","last_unstale":"2023-10-09T04:37:32.762983+0000","last_undegraded":"2023-10-09T04:37:32.762983+0000","last_fullsized":"2023-10-09T04:37:32.762983+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":395,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852637+0000","last_change":"2023-10-09T04:34:09.917744+0000","last_active":"2023-10-09T04:37:34.852637+0000","last_peered":"2023-10-09T04:37:34.852637+0000","last_clean":"2023-10-09T04:37:34.852637+0000","last_became_active":"2023-10-09T04:34:09.917258+0000","last_became_peered":"2023-10-09T04:34:09.917258+0000","last_unstale":"2023-10-09T04:37:34.852637+0000","last_undegraded":"2023-10-09T04:37:34.852637+0000","last_fullsized":"2023-10-09T04:37:34.852637+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":96,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761172+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:37:32.761172+0000","last_peered":"2023-10-09T04:37:32.761172+0000","last_clean":"2023-10-09T04:37:32.761172+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:37:32.761172+0000","last_undegraded":"2023-10-09T04:37:32.761172+0000","last_fullsized":"2023-10-09T04:37:32.761172+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.197850+0000","last_change":"2023-10-09T04:34:09.927188+0000","last_active":"2023-10-09T04:37:33.197850+0000","last_peered":"2023-10-09T04:37:33.197850+0000","last_clean":"2023-10-09T04:37:33.197850+0000","last_became_active":"2023-10-09T04:34:09.926564+0000","last_became_peered":"2023-10-09T04:34:09.926564+0000","last_unstale":"2023-10-09T04:37:33.197850+0000","last_undegraded":"2023-10-09T04:37:33.197850+0000","last_fullsized":"2023-10-09T04:37:33.197850+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"178'4200","reported_seq":4213,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:20.267341+0000","last_change":"2023-10-09T04:37:13.763373+0000","last_active":"2023-10-09T04:38:20.267341+0000","last_peered":"2023-10-09T04:38:20.267341+0000","last_clean":"2023-10-09T04:38:20.267341+0000","last_became_active":"2023-10-09T04:37:13.762984+0000","last_became_peered":"2023-10-09T04:37:13.762984+0000","last_unstale":"2023-10-09T04:38:20.267341+0000","last_undegraded":"2023-10-09T04:38:20.267341+0000","last_fullsized":"2023-10-09T04:38:20.267341+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4200,"log_dups_size":0,"ondisk_log_size":4200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275251200,"num_objects":4200,"num_object_clones":0,"num_object_copies":12600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4200,"num_write_kb":268800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":722,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852913+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:37:34.852913+0000","last_peered":"2023-10-09T04:37:34.852913+0000","last_clean":"2023-10-09T04:37:34.852913+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:37:34.852913+0000","last_undegraded":"2023-10-09T04:37:34.852913+0000","last_fullsized":"2023-10-09T04:37:34.852913+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756600+0000","last_change":"2023-10-09T04:36:24.632976+0000","last_active":"2023-10-09T04:37:32.756600+0000","last_peered":"2023-10-09T04:37:32.756600+0000","last_clean":"2023-10-09T04:37:32.756600+0000","last_became_active":"2023-10-09T04:36:24.632564+0000","last_became_peered":"2023-10-09T04:36:24.632564+0000","last_unstale":"2023-10-09T04:37:32.756600+0000","last_undegraded":"2023-10-09T04:37:32.756600+0000","last_fullsized":"2023-10-09T04:37:32.756600+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:11:16.527930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.757167+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:37:32.757167+0000","last_peered":"2023-10-09T04:37:32.757167+0000","last_clean":"2023-10-09T04:37:32.757167+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:37:32.757167+0000","last_undegraded":"2023-10-09T04:37:32.757167+0000","last_fullsized":"2023-10-09T04:37:32.757167+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"178'4128","reported_seq":4141,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:19.990566+0000","last_change":"2023-10-09T04:37:13.762982+0000","last_active":"2023-10-09T04:38:19.990566+0000","last_peered":"2023-10-09T04:38:19.990566+0000","last_clean":"2023-10-09T04:38:19.990566+0000","last_became_active":"2023-10-09T04:37:13.762606+0000","last_became_peered":"2023-10-09T04:37:13.762606+0000","last_unstale":"2023-10-09T04:38:19.990566+0000","last_undegraded":"2023-10-09T04:38:19.990566+0000","last_fullsized":"2023-10-09T04:38:19.990566+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4128,"log_dups_size":0,"ondisk_log_size":4128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270532608,"num_objects":4128,"num_object_clones":0,"num_object_copies":12384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4128,"num_write_kb":264192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":391,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853165+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:37:34.853165+0000","last_peered":"2023-10-09T04:37:34.853165+0000","last_clean":"2023-10-09T04:37:34.853165+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:37:34.853165+0000","last_undegraded":"2023-10-09T04:37:34.853165+0000","last_fullsized":"2023-10-09T04:37:34.853165+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.854241+0000","last_change":"2023-10-09T04:34:36.642995+0000","last_active":"2023-10-09T04:37:34.854241+0000","last_peered":"2023-10-09T04:37:34.854241+0000","last_clean":"2023-10-09T04:37:34.854241+0000","last_became_active":"2023-10-09T04:34:36.642548+0000","last_became_peered":"2023-10-09T04:34:36.642548+0000","last_unstale":"2023-10-09T04:37:34.854241+0000","last_undegraded":"2023-10-09T04:37:34.854241+0000","last_fullsized":"2023-10-09T04:37:34.854241+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:13.000173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.757037+0000","last_change":"2023-10-09T04:34:36.642199+0000","last_active":"2023-10-09T04:37:32.757037+0000","last_peered":"2023-10-09T04:37:32.757037+0000","last_clean":"2023-10-09T04:37:32.757037+0000","last_became_active":"2023-10-09T04:34:36.641868+0000","last_became_peered":"2023-10-09T04:34:36.641868+0000","last_unstale":"2023-10-09T04:37:32.757037+0000","last_undegraded":"2023-10-09T04:37:32.757037+0000","last_fullsized":"2023-10-09T04:37:32.757037+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:45.514132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"178'3890","reported_seq":3903,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:16.720000+0000","last_change":"2023-10-09T04:37:13.774838+0000","last_active":"2023-10-09T04:38:16.720000+0000","last_peered":"2023-10-09T04:38:16.720000+0000","last_clean":"2023-10-09T04:38:16.720000+0000","last_became_active":"2023-10-09T04:37:13.774474+0000","last_became_peered":"2023-10-09T04:37:13.774474+0000","last_unstale":"2023-10-09T04:38:16.720000+0000","last_undegraded":"2023-10-09T04:38:16.720000+0000","last_fullsized":"2023-10-09T04:38:16.720000+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":3890,"log_dups_size":0,"ondisk_log_size":3890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:10.659935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935040,"num_objects":3890,"num_object_clones":0,"num_object_copies":11670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3890,"num_write_kb":248960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":92,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853366+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:37:34.853366+0000","last_peered":"2023-10-09T04:37:34.853366+0000","last_clean":"2023-10-09T04:37:34.853366+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:37:34.853366+0000","last_undegraded":"2023-10-09T04:37:34.853366+0000","last_fullsized":"2023-10-09T04:37:34.853366+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":419,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.663811+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:37:55.663811+0000","last_peered":"2023-10-09T04:37:55.663811+0000","last_clean":"2023-10-09T04:37:55.663811+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:37:55.663811+0000","last_undegraded":"2023-10-09T04:37:55.663811+0000","last_fullsized":"2023-10-09T04:37:55.663811+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.852261+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:37:34.852261+0000","last_peered":"2023-10-09T04:37:34.852261+0000","last_clean":"2023-10-09T04:37:34.852261+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:37:34.852261+0000","last_undegraded":"2023-10-09T04:37:34.852261+0000","last_fullsized":"2023-10-09T04:37:34.852261+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"178'4081","reported_seq":4095,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:38:18.692559+0000","last_change":"2023-10-09T04:37:13.767844+0000","last_active":"2023-10-09T04:38:18.692559+0000","last_peered":"2023-10-09T04:38:18.692559+0000","last_clean":"2023-10-09T04:38:18.692559+0000","last_became_active":"2023-10-09T04:37:13.767535+0000","last_became_peered":"2023-10-09T04:37:13.767535+0000","last_unstale":"2023-10-09T04:38:18.692559+0000","last_undegraded":"2023-10-09T04:38:18.692559+0000","last_fullsized":"2023-10-09T04:38:18.692559+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":4081,"log_dups_size":0,"ondisk_log_size":4081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:31.233224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267452416,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4081,"num_write_kb":261184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":190,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.197794+0000","last_change":"2023-10-09T04:34:09.918839+0000","last_active":"2023-10-09T04:37:33.197794+0000","last_peered":"2023-10-09T04:37:33.197794+0000","last_clean":"2023-10-09T04:37:33.197794+0000","last_became_active":"2023-10-09T04:34:09.918201+0000","last_became_peered":"2023-10-09T04:34:09.918201+0000","last_unstale":"2023-10-09T04:37:33.197794+0000","last_undegraded":"2023-10-09T04:37:33.197794+0000","last_fullsized":"2023-10-09T04:37:33.197794+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":136,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756970+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:37:32.756970+0000","last_peered":"2023-10-09T04:37:32.756970+0000","last_clean":"2023-10-09T04:37:32.756970+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:37:32.756970+0000","last_undegraded":"2023-10-09T04:37:32.756970+0000","last_fullsized":"2023-10-09T04:37:32.756970+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.762027+0000","last_change":"2023-10-09T04:34:36.638499+0000","last_active":"2023-10-09T04:37:32.762027+0000","last_peered":"2023-10-09T04:37:32.762027+0000","last_clean":"2023-10-09T04:37:32.762027+0000","last_became_active":"2023-10-09T04:34:36.638089+0000","last_became_peered":"2023-10-09T04:34:36.638089+0000","last_unstale":"2023-10-09T04:37:32.762027+0000","last_undegraded":"2023-10-09T04:37:32.762027+0000","last_fullsized":"2023-10-09T04:37:32.762027+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":120,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.756712+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:37:32.756712+0000","last_peered":"2023-10-09T04:37:32.756712+0000","last_clean":"2023-10-09T04:37:32.756712+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:37:32.756712+0000","last_undegraded":"2023-10-09T04:37:32.756712+0000","last_fullsized":"2023-10-09T04:37:32.756712+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T04:37:55.665839+0000","last_change":"2023-10-09T04:34:36.644352+0000","last_active":"2023-10-09T04:37:55.665839+0000","last_peered":"2023-10-09T04:37:55.665839+0000","last_clean":"2023-10-09T04:37:55.665839+0000","last_became_active":"2023-10-09T04:34:36.643870+0000","last_became_peered":"2023-10-09T04:34:36.643870+0000","last_unstale":"2023-10-09T04:37:55.665839+0000","last_undegraded":"2023-10-09T04:37:55.665839+0000","last_fullsized":"2023-10-09T04:37:55.665839+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:32.488036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":103,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763335+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:37:32.763335+0000","last_peered":"2023-10-09T04:37:32.763335+0000","last_clean":"2023-10-09T04:37:32.763335+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:37:32.763335+0000","last_undegraded":"2023-10-09T04:37:32.763335+0000","last_fullsized":"2023-10-09T04:37:32.763335+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":25,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761779+0000","last_change":"2023-10-09T04:36:24.632928+0000","last_active":"2023-10-09T04:37:32.761779+0000","last_peered":"2023-10-09T04:37:32.761779+0000","last_clean":"2023-10-09T04:37:32.761779+0000","last_became_active":"2023-10-09T04:36:24.632560+0000","last_became_peered":"2023-10-09T04:36:24.632560+0000","last_unstale":"2023-10-09T04:37:32.761779+0000","last_undegraded":"2023-10-09T04:37:32.761779+0000","last_fullsized":"2023-10-09T04:37:32.761779+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.763701+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:37:32.763701+0000","last_peered":"2023-10-09T04:37:32.763701+0000","last_clean":"2023-10-09T04:37:32.763701+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:37:32.763701+0000","last_undegraded":"2023-10-09T04:37:32.763701+0000","last_fullsized":"2023-10-09T04:37:32.763701+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761547+0000","last_change":"2023-10-09T04:36:24.632650+0000","last_active":"2023-10-09T04:37:32.761547+0000","last_peered":"2023-10-09T04:37:32.761547+0000","last_clean":"2023-10-09T04:37:32.761547+0000","last_became_active":"2023-10-09T04:36:24.632235+0000","last_became_peered":"2023-10-09T04:36:24.632235+0000","last_unstale":"2023-10-09T04:37:32.761547+0000","last_undegraded":"2023-10-09T04:37:32.761547+0000","last_fullsized":"2023-10-09T04:37:32.761547+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":65,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:33.197497+0000","last_change":"2023-10-09T04:34:09.901145+0000","last_active":"2023-10-09T04:37:33.197497+0000","last_peered":"2023-10-09T04:37:33.197497+0000","last_clean":"2023-10-09T04:37:33.197497+0000","last_became_active":"2023-10-09T04:34:09.900787+0000","last_became_peered":"2023-10-09T04:34:09.900787+0000","last_unstale":"2023-10-09T04:37:33.197497+0000","last_undegraded":"2023-10-09T04:37:33.197497+0000","last_fullsized":"2023-10-09T04:37:33.197497+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:34.853928+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:37:34.853928+0000","last_peered":"2023-10-09T04:37:34.853928+0000","last_clean":"2023-10-09T04:37:34.853928+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:37:34.853928+0000","last_undegraded":"2023-10-09T04:37:34.853928+0000","last_fullsized":"2023-10-09T04:37:34.853928+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.757269+0000","last_change":"2023-10-09T04:34:36.645881+0000","last_active":"2023-10-09T04:37:32.757269+0000","last_peered":"2023-10-09T04:37:32.757269+0000","last_clean":"2023-10-09T04:37:32.757269+0000","last_became_active":"2023-10-09T04:34:36.644899+0000","last_became_peered":"2023-10-09T04:34:36.644899+0000","last_unstale":"2023-10-09T04:37:32.757269+0000","last_undegraded":"2023-10-09T04:37:32.757269+0000","last_fullsized":"2023-10-09T04:37:32.757269+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:11.020741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T04:37:32.761390+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:37:32.761390+0000","last_peered":"2023-10-09T04:37:32.761390+0000","last_clean":"2023-10-09T04:37:32.761390+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:37:32.761390+0000","last_undegraded":"2023-10-09T04:37:32.761390+0000","last_fullsized":"2023-10-09T04:37:32.761390+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4190568448,"num_objects":63943,"num_object_clones":0,"num_object_copies":191829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63943,"num_write_kb":4092352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12746424320,"data_stored":12746424320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63943,"ondisk_log_size":63943,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41128,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430567,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1627336,"kb_used_data":1326016,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92117816,"statfs":{"total":95995035648,"available":94328643584,"internally_reserved":0,"allocated":1357840384,"data_stored":1357296915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:00 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.1970000000000001,"15min":1.8999999999999999},"min":{"1min":0.33400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":7.4550000000000001,"5min":7.4550000000000001,"15min":250.65700000000001},"last":2.173},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.635,"15min":2.3250000000000002},"min":{"1min":0.313,"5min":0.27300000000000002,"15min":0.23400000000000001},"max":{"1min":2.5619999999999998,"5min":11.608000000000001,"15min":251.214},"last":0.67600000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.1719999999999999,"15min":1.0680000000000001},"min":{"1min":0.441,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.1160000000000001,"5min":5.9290000000000003,"15min":7.282},"last":2.5819999999999999},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.679,"15min":1.4059999999999999},"min":{"1min":0.48099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.879,"5min":11.555999999999999,"15min":11.988},"last":0.65500000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:38:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.5860000000000001,"15min":1.704},"min":{"1min":0.36499999999999999,"5min":0.33600000000000002,"15min":0.32300000000000001},"max":{"1min":3.8599999999999999,"5min":8.5120000000000005,"15min":106.456},"last":1.004},{"interface":"front","average":{"1min":1.004,"5min":1.389,"15min":1.6759999999999999},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.3460000000000001,"5min":7.4379999999999997,"15min":106.708},"last":4.7080000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.7989999999999999,"15min":1.3109999999999999},"min":{"1min":0.52600000000000002,"5min":0.39200000000000002,"15min":0.318},"max":{"1min":1.1659999999999999,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":4.6130000000000004},{"interface":"front","average":{"1min":1.014,"5min":1.595,"15min":1.2569999999999999},"min":{"1min":0.60899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.9370000000000001,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":4.6740000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.345,"15min":1.2669999999999999},"min":{"1min":0.625,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1899999999999999,"5min":11.683999999999999,"15min":11.683999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.876,"5min":1.341,"15min":1.1659999999999999},"min":{"1min":0.61299999999999999,"5min":0.317,"15min":0.317},"max":{"1min":1.1950000000000001,"5min":10.696999999999999,"15min":10.696999999999999},"last":0.93799999999999994}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.8360000000000001,"15min":1.9630000000000001},"min":{"1min":0.56899999999999995,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.278,"5min":8.0790000000000006,"15min":8.0790000000000006},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":2.052,"15min":2.2879999999999998},"min":{"1min":0.54900000000000004,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.395,"5min":10.786,"15min":10.786},"last":0.97199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.6919999999999999,"15min":1.716},"min":{"1min":0.52100000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.8570000000000002,"5min":11.955,"15min":11.955},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.335,"5min":1.212,"15min":1.1910000000000001},"min":{"1min":0.433,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.331,"5min":4.351,"15min":4.351},"last":2.2040000000000002}]}]},{"osd":6,"up_from":146,"seq":627065225267,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1592640,"kb_used_data":1266680,"kb_used_omap":5,"kb_used_meta":325946,"kb_avail":92152512,"statfs":{"total":95995035648,"available":94364172288,"internally_reserved":0,"allocated":1297080320,"data_stored":1296544916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":333769312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":3.1869999999999998,"15min":2.5910000000000002},"min":{"1min":0.42899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.1699999999999999,"5min":125.666,"15min":125.666},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.75800000000000001,"5min":3.1880000000000002,"15min":2.5819999999999999},"min":{"1min":0.46100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.3979999999999999,"5min":124.991,"15min":124.991},"last":1.7689999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":3.423,"15min":4.25},"min":{"1min":0.48399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.284,"5min":68.155000000000001,"15min":68.155000000000001},"last":1.6830000000000001},{"interface":"front","average":{"1min":0.84299999999999997,"5min":3.9889999999999999,"15min":4.9649999999999999},"min":{"1min":0.55100000000000005,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.4169999999999998,"5min":67.308999999999997,"15min":67.308999999999997},"last":2.742}]},{"osd":2,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":3.9449999999999998,"15min":5.0030000000000001},"min":{"1min":0.51200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.02,"5min":68.316000000000003,"15min":68.316000000000003},"last":1.8839999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":3.9750000000000001,"15min":4.8499999999999996},"min":{"1min":0.80300000000000005,"5min":0.313,"15min":0.313},"max":{"1min":1.228,"5min":68.210999999999999,"15min":68.210999999999999},"last":2.5790000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":2.125,"15min":2.1059999999999999},"min":{"1min":0.64800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.3280000000000001,"5min":11.673,"15min":11.673},"last":3.0139999999999998},{"interface":"front","average":{"1min":0.90300000000000002,"5min":2.3439999999999999,"15min":2.452},"min":{"1min":0.56699999999999995,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.5150000000000001,"5min":15.824,"15min":15.824},"last":2.9580000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.5049999999999999,"15min":1.629},"min":{"1min":0.59199999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.4870000000000001,"5min":8.0500000000000007,"15min":8.0500000000000007},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.5389999999999999,"15min":1.4990000000000001},"min":{"1min":0.628,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1579999999999999,"5min":10.324,"15min":10.324},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 04:37:43 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.073,"15min":2.4830000000000001},"min":{"1min":0.64100000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.113,"5min":10.193,"15min":10.193},"last":1.212},{"interface":"front","average":{"1min":1.129,"5min":2.1080000000000001,"15min":2.3599999999999999},"min":{"1min":0.65300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3660000000000001,"5min":11.727,"15min":11.727},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:37:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.9339999999999999,"15min":2.073},"min":{"1min":0.65700000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":2.5550000000000002,"5min":13.191000000000001,"15min":13.191000000000001},"last":1.236},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.661,"15min":1.75},"min":{"1min":0.78900000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.3180000000000001,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":0.98299999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958716,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1836984,"kb_used_data":1543984,"kb_used_omap":6,"kb_used_meta":292921,"kb_avail":91908168,"statfs":{"total":95995035648,"available":94113964032,"internally_reserved":0,"allocated":1581039616,"data_stored":1580279501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":299951116},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.4330000000000001,"15min":1.9359999999999999},"min":{"1min":0.30199999999999999,"5min":0.23599999999999999,"15min":0.22600000000000001},"max":{"1min":5.9320000000000004,"5min":13.220000000000001,"15min":158.435},"last":1.6319999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4059999999999999,"15min":2.052},"min":{"1min":0.373,"5min":0.218,"15min":0.218},"max":{"1min":6.9089999999999998,"5min":7.1210000000000004,"15min":159.38999999999999},"last":2.5339999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.2589999999999999,"15min":1.881},"min":{"1min":0.442,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":2.4630000000000001,"5min":9.4589999999999996,"15min":157.70400000000001},"last":1.6719999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.248,"15min":1.9299999999999999},"min":{"1min":0.47799999999999998,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":3.0270000000000001,"5min":9.1780000000000008,"15min":157.64099999999999},"last":2.4169999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:37:26 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":1.8799999999999999,"15min":1.409},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.27400000000000002},"max":{"1min":1.135,"5min":14.369,"15min":14.369},"last":1.6559999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.9019999999999999,"15min":1.345},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.249},"max":{"1min":0.92200000000000004,"5min":12.932,"15min":12.932},"last":4.1740000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.3089999999999999,"15min":1.2529999999999999},"min":{"1min":0.5,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":2.4950000000000001,"5min":12.327,"15min":12.327},"last":2.6709999999999998},{"interface":"front","average":{"1min":1.1990000000000001,"5min":1.256,"15min":1.3200000000000001},"min":{"1min":0.48399999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.0629999999999997,"5min":12.209,"15min":12.209},"last":1.7150000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.579,"15min":1.7949999999999999},"min":{"1min":0.48399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.0800000000000001,"5min":6.9900000000000002,"15min":6.9900000000000002},"last":2.5910000000000002},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.6379999999999999,"15min":1.8939999999999999},"min":{"1min":0.49099999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.649,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":2.8540000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6930000000000001,"15min":1.994},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.2329999999999997,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":2.6059999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.7410000000000001,"15min":2.0329999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.258,"5min":8.9879999999999995,"15min":8.9879999999999995},"last":2.5710000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.7480000000000002,"15min":1.734},"min":{"1min":0.44600000000000001,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":1.323,"5min":144.613,"15min":144.613},"last":2.7269999999999999},{"interface":"front","average":{"1min":1.2589999999999999,"5min":2.5329999999999999,"15min":1.7410000000000001},"min":{"1min":0.56100000000000005,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.1050000000000004,"5min":145.53700000000001,"15min":145.53700000000001},"last":1.6910000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122241,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307392,"kb_used_data":1983160,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":91437760,"statfs":{"total":95995035648,"available":93632266240,"internally_reserved":0,"allocated":2030755840,"data_stored":2029990296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":331995633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":1.982,"15min":1.3999999999999999},"min":{"1min":0.37,"5min":0.223,"15min":0.18099999999999999},"max":{"1min":0.81799999999999995,"5min":61.372999999999998,"15min":61.372999999999998},"last":4.3680000000000003},{"interface":"front","average":{"1min":0.62,"5min":1.5409999999999999,"15min":1.341},"min":{"1min":0.23000000000000001,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":1.038,"5min":25.931000000000001,"15min":43.091999999999999},"last":1.337}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.992,"15min":1.5940000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.6059999999999999,"5min":61.634999999999998,"15min":61.634999999999998},"last":4.3310000000000004},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.829,"15min":1.405},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":2.742,"5min":64.677000000000007,"15min":64.677000000000007},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 04:37:23 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":2.0649999999999999,"15min":1.2749999999999999},"min":{"1min":0.44,"5min":0.27400000000000002,"15min":0.20300000000000001},"max":{"1min":1.2909999999999999,"5min":64.622,"15min":64.622},"last":1.55},{"interface":"front","average":{"1min":0.67800000000000005,"5min":2.129,"15min":1.385},"min":{"1min":0.38700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":1.1140000000000001,"5min":64.540999999999997,"15min":64.540999999999997},"last":2.069}]},{"osd":4,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.909,"15min":1.5269999999999999},"min":{"1min":0.34200000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.9960000000000004,"5min":61.549999999999997,"15min":61.549999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.7529999999999999,"15min":1.3109999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.308},"max":{"1min":1.702,"5min":61.902999999999999,"15min":61.902999999999999},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.8809999999999998,"15min":3.633},"min":{"1min":0.47199999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.4850000000000001,"5min":61.972999999999999,"15min":61.972999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.95599999999999996,"5min":3.419,"15min":4.2930000000000001},"min":{"1min":0.56799999999999995,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.425,"5min":62.417000000000002,"15min":62.417000000000002},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":2.7440000000000002,"15min":3.6930000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.2990000000000004,"5min":62.462000000000003,"15min":62.462000000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.9369999999999998,"15min":3.9750000000000001},"min":{"1min":0.44400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6760000000000002,"5min":61.808,"15min":61.808},"last":0.61499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.056,"15min":1.6910000000000001},"min":{"1min":0.504,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6189999999999998,"5min":62.006,"15min":62.006},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.347,"5min":2.1699999999999999,"15min":1.734},"min":{"1min":0.47199999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0359999999999996,"5min":61.606999999999999,"15min":61.606999999999999},"last":0.41599999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795190,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818932,"kb_used_data":1507500,"kb_used_omap":11,"kb_used_meta":311412,"kb_avail":91926220,"statfs":{"total":95995035648,"available":94132449280,"internally_reserved":0,"allocated":1543680000,"data_stored":1542923542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":318885930},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.512,"15min":1.1830000000000001},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.191},"max":{"1min":7.2789999999999999,"5min":18.155999999999999,"15min":18.155999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.518,"15min":1.228},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20599999999999999},"max":{"1min":1.6539999999999999,"5min":14.548,"15min":14.548},"last":3.1520000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.25},"max":{"1min":3.024,"5min":14.438000000000001,"15min":14.438000000000001},"last":2.173},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.3520000000000001,"15min":1.252},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.1859999999999999,"5min":13.728,"15min":13.728},"last":3.1859999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.796,"15min":1.2609999999999999},"min":{"1min":0.41099999999999998,"5min":0.29599999999999999,"15min":0.27800000000000002},"max":{"1min":4.5279999999999996,"5min":14.515000000000001,"15min":14.515000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.782,"15min":1.349},"min":{"1min":0.41199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.4180000000000001,"5min":14.66,"15min":14.66},"last":0.95599999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.371,"15min":1.339},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.29599999999999999},"max":{"1min":1.272,"5min":13.689,"15min":13.689},"last":1.0580000000000001},{"interface":"front","average":{"1min":2.028,"5min":1.704,"15min":1.5529999999999999},"min":{"1min":0.625,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":10.002000000000001,"5min":13.759,"15min":13.759},"last":1.512}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.5899999999999999,"15min":3.0600000000000001},"min":{"1min":0.46200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.057,"5min":14.564,"15min":14.564},"last":1.601},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.8029999999999999,"15min":2.0169999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.272,"5min":14.305999999999999,"15min":14.305999999999999},"last":2.0779999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:37:16 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6379999999999999,"15min":1.6279999999999999},"min":{"1min":0.46000000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2570000000000001,"5min":9.5719999999999992,"15min":9.5719999999999992},"last":2.1179999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.548,"15min":1.4159999999999999},"min":{"1min":0.69299999999999995,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.4430000000000001,"5min":14.683999999999999,"15min":14.683999999999999},"last":1.0009999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.532,"15min":1.3839999999999999},"min":{"1min":0.54000000000000004,"5min":0.36199999999999999,"15min":0.35099999999999998},"max":{"1min":2.8820000000000001,"5min":14.372999999999999,"15min":14.372999999999999},"last":2.1480000000000001},{"interface":"front","average":{"1min":1.111,"5min":1.577,"15min":1.425},"min":{"1min":0.56699999999999995,"5min":0.38400000000000001,"15min":0.29899999999999999},"max":{"1min":2.3820000000000001,"5min":14.6,"15min":14.6},"last":1.032}]}]},{"osd":3,"up_from":103,"seq":442381631662,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2676728,"kb_used_data":2345712,"kb_used_omap":5,"kb_used_meta":331002,"kb_avail":91068424,"statfs":{"total":95995035648,"available":93254066176,"internally_reserved":0,"allocated":2402009088,"data_stored":2401248876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":338946222},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.2150000000000001,"15min":1.409},"min":{"1min":0.249,"5min":0.154,"15min":0.154},"max":{"1min":0.79400000000000004,"5min":9.6379999999999999,"15min":61.966999999999999},"last":1.032},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.3759999999999999,"15min":1.4630000000000001},"min":{"1min":0.36399999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.88100000000000001,"5min":12.638999999999999,"15min":62.052999999999997},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.2170000000000001,"15min":1.238},"min":{"1min":0.317,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.77900000000000003,"5min":11.920999999999999,"15min":61.395000000000003},"last":1.9119999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.1539999999999999,"15min":1.2150000000000001},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.19500000000000001},"max":{"1min":0.94699999999999995,"5min":5.5839999999999996,"15min":61.179000000000002},"last":1.0029999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.6899999999999999,"15min":2.0649999999999999},"min":{"1min":0.38900000000000001,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":0.99399999999999999,"5min":12.087999999999999,"15min":181.959},"last":2.976},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1.4790000000000001,"15min":1.9930000000000001},"min":{"1min":0.34399999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.002,"5min":6.9130000000000003,"15min":181.91200000000001},"last":2.673}]},{"osd":4,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.1930000000000001,"15min":1.0649999999999999},"min":{"1min":0.44600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":1.0649999999999999,"5min":7.3520000000000003,"15min":7.3520000000000003},"last":2.339},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3089999999999999,"15min":1.155},"min":{"1min":0.496,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.2509999999999999,"5min":7.9260000000000002,"15min":7.9260000000000002},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.542,"15min":1.7749999999999999},"min":{"1min":0.52500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1679999999999999,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.88500000000000001,"5min":2.0590000000000002,"15min":2.4729999999999999},"min":{"1min":0.48899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.325,"5min":7.976,"15min":7.976},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:37:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":2.2869999999999999,"15min":2.4289999999999998},"min":{"1min":0.56999999999999995,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.22,"5min":11.842000000000001,"15min":11.842000000000001},"last":2.0800000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.7330000000000001,"15min":1.7949999999999999},"min":{"1min":0.63700000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2270000000000001,"5min":8.5020000000000007,"15min":8.5020000000000007},"last":2.3170000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":4.3849999999999998,"15min":2.2029999999999998},"min":{"1min":0.68000000000000005,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.292,"5min":198.012,"15min":198.012},"last":2.1110000000000002},{"interface":"front","average":{"1min":0.98299999999999998,"5min":4.0629999999999997,"15min":2.117},"min":{"1min":0.61899999999999999,"5min":0.47399999999999998,"15min":0.34599999999999997},"max":{"1min":1.3280000000000001,"5min":198.05199999999999,"15min":198.05199999999999},"last":1.1739999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019264,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1644672,"kb_used_data":1331064,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92100480,"statfs":{"total":95995035648,"available":94310891520,"internally_reserved":0,"allocated":1363009536,"data_stored":1362473645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1679999999999999,"15min":1.379},"min":{"1min":0.50700000000000001,"5min":0.26100000000000001,"15min":0.24099999999999999},"max":{"1min":4.9080000000000004,"5min":12.295999999999999,"15min":48.450000000000003},"last":0.308},{"interface":"front","average":{"1min":1.665,"5min":1.502,"15min":1.581},"min":{"1min":0.56699999999999995,"5min":0.31,"15min":0.28599999999999998},"max":{"1min":8.5990000000000002,"5min":9.1509999999999998,"15min":49.283000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.196,"15min":1.514},"min":{"1min":0.308,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":4.9569999999999999,"5min":10.459,"15min":79.361999999999995},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.2490000000000001,"15min":1.639},"min":{"1min":0.48199999999999998,"5min":0.316,"15min":0.29199999999999998},"max":{"1min":5.3929999999999998,"5min":10.487,"15min":79.292000000000002},"last":0.83199999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.2480000000000002,"15min":1.754},"min":{"1min":0.70899999999999996,"5min":0.31900000000000001,"15min":0.31},"max":{"1min":2.6930000000000001,"5min":77.537000000000006,"15min":77.537000000000006},"last":0.747},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.907,"15min":1.796},"min":{"1min":0.56200000000000006,"5min":0.36099999999999999,"15min":0.311},"max":{"1min":1.2390000000000001,"5min":77.188000000000002,"15min":77.188000000000002},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:37:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.532,"15min":1.2210000000000001},"min":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":1.456,"5min":10.417,"15min":10.417},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.54,"15min":1.319},"min":{"1min":0.53300000000000003,"5min":0.34599999999999997,"15min":0.32000000000000001},"max":{"1min":1.391,"5min":8.7899999999999991,"15min":11.112},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:37:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.1479999999999999,"15min":1.105},"min":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2549999999999999,"5min":10.412000000000001,"15min":10.412000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.6180000000000001,"15min":1.712},"min":{"1min":0.77200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":2.6480000000000001,"5min":6.4470000000000001,"15min":6.4470000000000001},"last":0.79800000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.05,"15min":0.92000000000000004},"min":{"1min":0.42799999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.8340000000000001,"5min":3.8340000000000001,"15min":3.8340000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.2390000000000001,"15min":1.2529999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.635,"5min":10.257999999999999,"15min":10.257999999999999},"last":0.52500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.0629999999999999,"15min":1.216},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.526,"5min":8.734,"15min":11.725},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.413,"15min":1.2490000000000001},"min":{"1min":0.42899999999999999,"5min":0.42399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6150000000000002,"5min":13.349,"15min":13.349},"last":0.60299999999999998}]}]},{"osd":5,"up_from":150,"seq":644245094446,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1395056,"kb_used_data":1053544,"kb_used_omap":6,"kb_used_meta":341497,"kb_avail":92350096,"statfs":{"total":95995035648,"available":94566498304,"internally_reserved":0,"allocated":1078829056,"data_stored":1078304997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":349693203},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.7050000000000001,"15min":1.7929999999999999},"min":{"1min":0.53900000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.0339999999999998,"5min":7.0339999999999998,"15min":7.0339999999999998},"last":0.998},{"interface":"front","average":{"1min":0.76900000000000002,"5min":1.873,"15min":2.0390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.389,"5min":8.2319999999999993,"15min":8.2319999999999993},"last":2.8250000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.9039999999999999,"15min":2.169},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.2530000000000001,"5min":7.2750000000000004,"15min":7.2750000000000004},"last":2.8519999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7030000000000001,"15min":1.8129999999999999},"min":{"1min":0.58999999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.1930000000000001,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":2.9740000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":2.1499999999999999,"15min":2.4020000000000001},"min":{"1min":0.49299999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.6200000000000001,"5min":12.69,"15min":12.69},"last":9.1449999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.0329999999999999,"15min":2.1989999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.28,"5min":10.657999999999999,"15min":10.657999999999999},"last":2.9900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.694,"15min":1.8799999999999999},"min":{"1min":0.46400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3240000000000001,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":4.6550000000000002},{"interface":"front","average":{"1min":0.94599999999999995,"5min":2.25,"15min":2.6600000000000001},"min":{"1min":0.60599999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.3520000000000001,"5min":8.2859999999999996,"15min":8.2859999999999996},"last":4.8680000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.738,"15min":1.972},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.4319999999999999,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":2.7309999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.4330000000000001,"15min":1.5720000000000001},"min":{"1min":0.51700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.4119999999999999,"5min":6.8300000000000001,"15min":6.8300000000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.649,"15min":1.833},"min":{"1min":0.71199999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.569,"5min":11.087,"15min":11.087},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.8140000000000001,"15min":2.0600000000000001},"min":{"1min":0.47299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":2.7869999999999999,"5min":9.3659999999999997,"15min":9.3659999999999997},"last":2.891}]},{"osd":7,"last update":"Mon Oct 9 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":2.3290000000000002,"15min":2.3340000000000001},"min":{"1min":0.48299999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.6759999999999999,"5min":39.771000000000001,"15min":39.771000000000001},"last":2.9100000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":2.746,"15min":3.0579999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.506,"5min":39.905000000000001,"15min":39.905000000000001},"last":0.52000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2054160384,"data_stored":2054160384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1586954240,"data_stored":1586954240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1550647296,"data_stored":1550647296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2429878272,"data_stored":2429878272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1364000768,"data_stored":1364000768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1082261504,"data_stored":1082261504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1310261248,"data_stored":1310261248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1368260608,"data_stored":1368260608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:38:25.358 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:38:25.359 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:38:25.359 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.616730099432939 2023-10-09T04:38:25.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 0.616730099432939 2023-10-09T04:38:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:25.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:25.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:25.490 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:25.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:25.625 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:25.647 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:38:25.697 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:38:25.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:25.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:38:25.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:25.770 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:38:25.808 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:38:25.849 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:38:25.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:25.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:25 smithi153 bash[59565]: cluster 2023-10-09T04:38:24.452225+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:38:25.869 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:25 smithi153 bash[59565]: audit 2023-10-09T04:38:25.602437+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.82:0/1623153966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:25.884 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:38:25.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:25.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:25.929 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:38:25.936 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.936 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:38:25.973 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:25 smithi082 bash[77063]: cluster 2023-10-09T04:38:24.452225+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:38:25.973 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:25 smithi082 bash[77063]: audit 2023-10-09T04:38:25.602437+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.82:0/1623153966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:25.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:25 smithi082 bash[79733]: cluster 2023-10-09T04:38:24.452225+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:38:25.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:25 smithi082 bash[79733]: audit 2023-10-09T04:38:25.602437+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.82:0/1623153966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:25.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:26.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 72256 72240 90.2838 57.1875 0.00821748 0.0110711 2023-10-09T04:38:26.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:26.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:26.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:38:26.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:26.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:26.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:26.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:27.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:27.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:27.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 73095 73079 89.5417 52.4375 0.0190506 0.0111623 2023-10-09T04:38:27.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:27.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:27.302 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:27.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:27.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:27.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:27.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:27.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:27.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:27.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:27.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:27.799 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:27 smithi153 bash[59565]: cluster 2023-10-09T04:38:26.452923+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:38:27.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:27.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:27.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:27.926 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:27.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:28.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 73846 73830 88.7224 46.9375 0.0147363 0.0112649 2023-10-09T04:38:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:27 smithi082 bash[77063]: cluster 2023-10-09T04:38:26.452923+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:38:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:27 smithi082 bash[79733]: cluster 2023-10-09T04:38:26.452923+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:38:28.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:28.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:28.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:28.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.532 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:28.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:28.665 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:28.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:28.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:28.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:28.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:28.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:29.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:29.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:29.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:29.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 74917 74901 88.3113 66.9375 0.00759556 0.011317 2023-10-09T04:38:29.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:29.213 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:29.316 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:29.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:29.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:29.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:29.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:29.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:29.733 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:38:29.733 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:38:29.733 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:75429 7.9 e179/173 7:916986f9:::benchmark_data_smithi082_155217_object75428:head v 179'4722, mlcod=179'4720)", 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.709899+0000", 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.017071032, 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017101937000000001, 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.734 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709899+0000", 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709899+0000", 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.735 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709898+0000", 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709920+0000", 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2327000000000002e-05 2023-10-09T04:38:29.736 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709930+0000", 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5550000000000008e-06 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.709936+0000", 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7189999999999998e-06 2023-10-09T04:38:29.737 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.713420+0000", 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034841410000000001 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.713439+0000", 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9426e-05 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.738 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:75438 7.b 7:d8b7804f:::benchmark_data_smithi082_155217_object75437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.710474+0000", 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.016496015999999999, 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.016605466999999999, 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:38:29.739 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 75438 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710474+0000", 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.740 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710474+0000", 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710472+0000", 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710504+0000", 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1479000000000001e-05 2023-10-09T04:38:29.741 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710522+0000", 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8431999999999999e-05 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.710530+0000", 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7600000000000002e-06 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.742 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.713149+0000", 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0026191249999999999 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.713294+0000", 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000144893 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.743 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.713448+0000", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015395599999999999 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.723620+0000", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010171537 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.723819+0000", 2023-10-09T04:38:29.744 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019900399999999999 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:75440 7.b 7:da17c064:::benchmark_data_smithi082_155217_object75439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.716464+0000", 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.010505512999999999, 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010717720999999999, 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:38:29.745 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 75440 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716464+0000", 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.746 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716464+0000", 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716463+0000", 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.747 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716480+0000", 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6620999999999999e-05 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716487+0000", 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9949999999999999e-06 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.716534+0000", 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6956999999999998e-05 2023-10-09T04:38:29.748 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.723847+0000", 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0073136080000000001 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724029+0000", 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018169700000000001 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.749 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724186+0000", 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015727699999999999 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:75443 7.d e179/173 7:b41ccfcc:::benchmark_data_smithi082_155217_object75442:head v 179'4657, mlcod=179'4655)", 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.717671+0000", 2023-10-09T04:38:29.750 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0092984369999999997, 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0095887330000000003, 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717671+0000", 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.751 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717671+0000", 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717671+0000", 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.752 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717714+0000", 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3541000000000002e-05 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717722+0000", 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3559999999999993e-06 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.753 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717728+0000", 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8799999999999996e-06 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.718193+0000", 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00046475299999999999 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.718213+0000", 2023-10-09T04:38:29.754 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9675000000000001e-05 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:75444 7.0 e179/173 7:0ca993be:::benchmark_data_smithi082_155217_object75443:head v 179'4718, mlcod=179'4716)", 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.717787+0000", 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0091828499999999993, 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0095397439999999993, 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:29.755 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717787+0000", 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717787+0000", 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.756 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717784+0000", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999967 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717835+0000", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1446000000000002e-05 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717846+0000", 2023-10-09T04:38:29.757 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1004e-05 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.717854+0000", 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4660000000000002e-06 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.721246+0000", 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0033924179999999999 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.758 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.721269+0000", 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3102e-05 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.727306+0000", 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060365130000000003 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.759 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:75449 7.9 e179/173 7:9009d638:::benchmark_data_smithi082_155217_object75448:head v 179'4723, mlcod=179'4720)", 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:38:29.724509+0000", 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0024607470000000001, 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028873689999999999, 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:38:29.760 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724509+0000", 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724509+0000", 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724508+0000", 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:38:29.761 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724531+0000", 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3119e-05 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724542+0000", 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1476e-05 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.762 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724549+0000", 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.007e-06 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724907+0000", 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00035772700000000001 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:38:29.763 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:38:29.724926+0000", 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8700999999999999e-05 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 6 2023-10-09T04:38:29.764 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:38:29.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:29.793 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:29.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:29.907 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[77063]: cluster 2023-10-09T04:38:28.453484+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:38:29.907 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[77063]: audit 2023-10-09T04:38:29.278366+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:29.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[77063]: audit 2023-10-09T04:38:29.279485+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:29.908 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:29] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-10-09T04:38:29.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[79733]: cluster 2023-10-09T04:38:28.453484+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:38:29.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[79733]: audit 2023-10-09T04:38:29.278366+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:29.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:29 smithi082 bash[79733]: audit 2023-10-09T04:38:29.279485+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:29.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:29.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:30.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:30.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:29 smithi153 bash[59565]: cluster 2023-10-09T04:38:28.453484+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:38:30.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:29 smithi153 bash[59565]: audit 2023-10-09T04:38:29.278366+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:30.041 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:29 smithi153 bash[59565]: audit 2023-10-09T04:38:29.279485+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:30.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:30.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:30.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:30.171 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 75894 75878 87.8066 61.0625 0.0110534 0.0113829 2023-10-09T04:38:30.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:30.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:30.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:30.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:30.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:30.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:30.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:38:31.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 76891 76875 87.343 62.3125 0.0159845 0.0114428 2023-10-09T04:38:31.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:31.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:38:31.296 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:31.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:31.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:31.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:31.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:31.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:31.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:31.743 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:31.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:31.861 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:31 smithi082 bash[77063]: cluster 2023-10-09T04:38:30.454772+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T04:38:31.861 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:31 smithi082 bash[79733]: cluster 2023-10-09T04:38:30.454772+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T04:38:31.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:31.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:31.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:32.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:32.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:31 smithi153 bash[59565]: cluster 2023-10-09T04:38:30.454772+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T04:38:32.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:32.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 77725 77709 86.714 52.125 0.0160305 0.011526 2023-10-09T04:38:32.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.292 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:32.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:32.421 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:32.523 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:32.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:32.650 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:32.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:32.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:32.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:32.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:32.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:33.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:33.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:33.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 78684 78668 86.2442 59.9375 0.0115594 0.0115895 2023-10-09T04:38:33.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:33.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:33.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:33.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:33.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:33.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:33.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:33.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:33.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:33.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:33.801 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:33.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:33.810 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:33.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:33 smithi082 bash[77063]: cluster 2023-10-09T04:38:32.455350+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T04:38:33.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:33 smithi082 bash[79733]: cluster 2023-10-09T04:38:32.455350+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T04:38:34.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:34.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:34.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 79595 79579 85.7389 56.9375 0.0278476 0.0116573 2023-10-09T04:38:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:33 smithi153 bash[59565]: cluster 2023-10-09T04:38:32.455350+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T04:38:34.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:34.218 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:34.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:34.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:34.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:34.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:34.482 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:34.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:34.619 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:34.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:34.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:34.750 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:34.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:34.882 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:34.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:34.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.003 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:35.106 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:35.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:35.172 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 80344 80328 85.0791 46.8125 0.00368782 0.0117024 2023-10-09T04:38:35.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:35.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:35.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:35.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:35.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:35.704 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:35.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:35.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:35.838 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:35.838 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:35.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:35 smithi153 bash[59565]: cluster 2023-10-09T04:38:34.455994+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T04:38:36.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:35 smithi082 bash[79733]: cluster 2023-10-09T04:38:34.455994+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T04:38:36.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:35 smithi082 bash[77063]: cluster 2023-10-09T04:38:34.455994+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2023-10-09T04:38:36.299 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:38:36.171690+0000 min lat: 0.00131815 max lat: 0.308439 avg lat: 0.0118069 2023-10-09T04:38:36.299 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:36.300 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 81292 81276 84.6485 59.25 0.00763577 0.0118069 2023-10-09T04:38:36.611 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:36.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:36.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:36.744 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:36.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:36.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:36.878 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:36.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:36.986 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:37.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:37.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:37.381 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 82291 82275 84.1091 62.4375 0.0239018 0.011884 2023-10-09T04:38:37.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:37.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:37.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:37.682 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:37.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:37.798 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:37 smithi153 bash[59565]: cluster 2023-10-09T04:38:36.456576+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:38:37.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:37.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:37.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:37.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:37 smithi082 bash[77063]: cluster 2023-10-09T04:38:36.456576+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:38:37.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:37 smithi082 bash[79733]: cluster 2023-10-09T04:38:36.456576+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T04:38:38.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:38.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:38.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:38.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:38.381 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 83103 83087 83.4621 50.75 0.0192806 0.011976 2023-10-09T04:38:38.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:38.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:38.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:38.716 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:38.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:38.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:38.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:38.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:38.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:38.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:38.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:39.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:39.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:39.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:39.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:39.381 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 84041 84025 83.0691 58.625 0.0104571 0.0120327 2023-10-09T04:38:39.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:39.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:39.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:39.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.722 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:39.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:39.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:39.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:39.899 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:39 smithi082 bash[77063]: cluster 2023-10-09T04:38:38.457134+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:38:39.899 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:39] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.33.4" 2023-10-09T04:38:39.900 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:39 smithi082 bash[79733]: cluster 2023-10-09T04:38:38.457134+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:38:39.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:39.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:39.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:40.100 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:40.112 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:39 smithi153 bash[59565]: cluster 2023-10-09T04:38:38.457134+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:38:40.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:40.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:40.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.381 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 84933 84917 82.6436 55.75 0.036359 0.0120945 2023-10-09T04:38:40.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:40.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:40.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:40.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:40.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:40.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:40.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:40.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:41.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:41.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:41.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:41.302 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:41.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:41.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 85901 85885 82.304 60.5 0.0122831 0.0121452 2023-10-09T04:38:41.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:41.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:41.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:41.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:41.790 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:41 smithi082 bash[77063]: cluster 2023-10-09T04:38:40.457728+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:38:41.790 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:41 smithi082 bash[79733]: cluster 2023-10-09T04:38:40.457728+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:38:41.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.802 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:41 smithi153 bash[59565]: cluster 2023-10-09T04:38:40.457728+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:38:41.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:41.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.952 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:41.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:42.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:42.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:42.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:42.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:42.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:42.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 86737 86721 81.85 52.25 0.0193075 0.0122115 2023-10-09T04:38:42.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:42.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:42.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:42.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:42.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:42.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:42.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:42.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:42.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:42.901 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:42.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:43.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:43.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:43.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:43.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:43.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:43.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 87610 87594 81.444 54.5625 0.0135144 0.0122734 2023-10-09T04:38:43.558 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:43.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:43.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:43.794 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:43 smithi082 bash[77063]: cluster 2023-10-09T04:38:42.458316+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:38:43.794 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:43 smithi082 bash[79733]: cluster 2023-10-09T04:38:42.458316+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:38:43.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:43.830 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:43.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:43.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:43.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:44.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:44.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:43 smithi153 bash[59565]: cluster 2023-10-09T04:38:42.458316+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:38:44.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:44.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:44.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:44.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 88506 88490 81.0709 56 0.0272135 0.012329 2023-10-09T04:38:44.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:44.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:44.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:44.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:44.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:44.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:44 smithi082 bash[77063]: audit 2023-10-09T04:38:43.960874+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:44.927 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:44 smithi082 bash[77063]: audit 2023-10-09T04:38:44.274248+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:44.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:44 smithi082 bash[79733]: audit 2023-10-09T04:38:43.960874+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:44.927 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:44 smithi082 bash[79733]: audit 2023-10-09T04:38:44.274248+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:44.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:45.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:45.057 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:45.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:45.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:44 smithi153 bash[59565]: audit 2023-10-09T04:38:43.960874+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:45.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:44 smithi153 bash[59565]: audit 2023-10-09T04:38:44.274248+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:45.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:45.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:45.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:45.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 16 89411 89395 80.7167 56.5625 0.0176492 0.0123839 2023-10-09T04:38:45.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:45.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:45.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:45.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:45.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:45.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.743 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:45.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:45.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:45.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:45.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:45.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:45.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:45 smithi082 bash[77063]: cluster 2023-10-09T04:38:44.458857+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:38:45.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:45 smithi082 bash[79733]: cluster 2023-10-09T04:38:44.458857+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:38:46.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:46.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:46.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:45 smithi153 bash[59565]: cluster 2023-10-09T04:38:44.458857+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s 2023-10-09T04:38:46.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:46.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:46.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:46.382 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 90323 90307 80.3789 57 0.0168406 0.0124332 2023-10-09T04:38:46.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:46.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:46.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:46.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:46.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:46.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:46.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:46.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:46.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:46.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:46.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:47.054 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:47.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:47.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:47.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:47.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:47.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:47.385 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 16 91183 91167 80.001 53.75 0.026905 0.0124944 2023-10-09T04:38:47.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:47.524 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:47.613 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:38:47.613 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:38:47.613 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:38:47.613 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:38:47.613 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:38:47.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:47.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:47.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:47.794 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:47.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:47.910 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:47 smithi153 bash[59565]: cluster 2023-10-09T04:38:46.459424+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:38:47.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:47.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:47.966 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:38:47.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:47 smithi082 bash[77063]: cluster 2023-10-09T04:38:46.459424+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:38:47.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:47 smithi082 bash[79733]: cluster 2023-10-09T04:38:46.459424+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:38:48.016 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:38:48.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:48.055 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:38:48.092 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:38:48.141 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:38:48.183 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:38:48.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:48.221 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:38:48.258 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:38:48.266 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.267 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.268 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:38:48.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:48.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:48.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 16 91845 91829 79.4661 41.375 0.0223635 0.012577 2023-10-09T04:38:48.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:48.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:48.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:48.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:48.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:48.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:48.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:48.831 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 0.61673 (9de2) 2023-10-09T04:38:48.834 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:38:48.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[79733]: audit 2023-10-09T04:38:47.823550+0000 mon.a (mon.0) 1070 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]: dispatch 2023-10-09T04:38:48.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[79733]: audit 2023-10-09T04:38:47.914131+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.82:0/109771269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:48.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[79733]: audit 2023-10-09T04:38:48.211791+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.82:0/565817023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:48.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[79733]: audit 2023-10-09T04:38:48.212586+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:48.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:48.904 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:48.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:49.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:49.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:49.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:49.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[77063]: audit 2023-10-09T04:38:47.823550+0000 mon.a (mon.0) 1070 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]: dispatch 2023-10-09T04:38:49.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[77063]: audit 2023-10-09T04:38:47.914131+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.82:0/109771269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:49.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[77063]: audit 2023-10-09T04:38:48.211791+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.82:0/565817023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:48 smithi082 bash[77063]: audit 2023-10-09T04:38:48.212586+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:49.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:48 smithi153 bash[59565]: audit 2023-10-09T04:38:47.823550+0000 mon.a (mon.0) 1070 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]: dispatch 2023-10-09T04:38:49.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:48 smithi153 bash[59565]: audit 2023-10-09T04:38:47.914131+0000 mon.a (mon.0) 1071 : audit 0 from='client.? 172.21.15.82:0/109771269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:49.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:48 smithi153 bash[59565]: audit 2023-10-09T04:38:48.211791+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.82:0/565817023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:48 smithi153 bash[59565]: audit 2023-10-09T04:38:48.212586+0000 mon.a (mon.0) 1072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:49.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:49.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:49.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 92303 92287 78.7717 28.625 0.00390742 0.0126021 2023-10-09T04:38:49.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:49.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:49.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:49.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:49.682 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:49.785 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:49.807 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:49.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:49.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:49.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:49] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-10-09T04:38:49.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.460003+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T04:38:49.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.814206+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.816718+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: audit 2023-10-09T04:38:48.821838+0000 mon.a (mon.0) 1074 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]': finished 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: audit 2023-10-09T04:38:48.821902+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.822065+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.822880+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.825711+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:48.826263+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.665667+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:38:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.460003+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.814206+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.816718+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: audit 2023-10-09T04:38:48.821838+0000 mon.a (mon.0) 1074 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]': finished 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: audit 2023-10-09T04:38:48.821902+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.822065+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.822880+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.825711+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:48.826263+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:49.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:49 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.665667+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:38:50.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:50.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:50.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.460003+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.814206+0000 mon.a (mon.0) 1073 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.816718+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: audit 2023-10-09T04:38:48.821838+0000 mon.a (mon.0) 1074 : audit 1 from='client.? 172.21.15.82:0/2112963143' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.616730099432939}]': finished 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: audit 2023-10-09T04:38:48.821902+0000 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.822065+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.822880+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.825711+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:48.826263+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T04:38:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:49 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.665667+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:38:50.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:50.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:50.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:50.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 92303 92287 77.7103 0 - 0.0126021 2023-10-09T04:38:50.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:50.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:50.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:50.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:50.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:50.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:50.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:50.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:50.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:50.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:50.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:50.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:51.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.665774+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.665974+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.667565+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.670624+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.670869+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.671202+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.671442+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.673667+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:49.673793+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: audit 2023-10-09T04:38:50.207386+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: audit 2023-10-09T04:38:50.213771+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.668841+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.670239+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:38:51.084 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:50 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.670878+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.098 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:51.201 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.665774+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.665974+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.667565+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.212 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.670624+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.213 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.670869+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.213 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.671202+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.213 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.671442+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.213 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.673667+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.213 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:49.673793+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.214 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: audit 2023-10-09T04:38:50.207386+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.214 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: audit 2023-10-09T04:38:50.213771+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.214 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.668841+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.214 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.670239+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:38:51.215 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.670878+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.665774+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.665974+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.667565+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.670624+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.670869+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.671202+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.671442+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.673667+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:49.673793+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T04:38:51.216 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: audit 2023-10-09T04:38:50.207386+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: audit 2023-10-09T04:38:50.213771+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:51.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.668841+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.670239+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:38:51.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:50 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.670878+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:38:51.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:51.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.375 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:51.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 92608 92592 76.9306 9.53125 0.0185171 0.0129922 2023-10-09T04:38:51.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:51.508 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:51.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:51.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:51.753 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:51.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:51.889 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.460964+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.665846+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.708520+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.711433+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[77063]: cluster 2023-10-09T04:38:50.878242+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.460964+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:38:51.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.665846+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.708520+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.711433+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:51 smithi082 bash[79733]: cluster 2023-10-09T04:38:50.878242+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:51.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:51.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:52.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:52.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:52.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:52.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:51 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.460964+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:38:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:51 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.665846+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:51 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.708520+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:51 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.711433+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:51 smithi153 bash[59565]: cluster 2023-10-09T04:38:50.878242+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T04:38:52.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:52.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:52.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:52.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 93351 93335 76.5304 46.4375 0.0733735 0.013059 2023-10-09T04:38:52.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:52.456 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:52.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:52.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:52.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:52.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:52.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:52.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:52.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:52.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:52.996 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:52.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:53.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:53.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:52 smithi153 bash[59565]: cluster 2023-10-09T04:38:52.836128+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: Degraded data redundancy: 5803/276867 objects degraded (2.096%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:38:53.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:53.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:52 smithi082 bash[79733]: cluster 2023-10-09T04:38:52.836128+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: Degraded data redundancy: 5803/276867 objects degraded (2.096%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:38:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:52 smithi082 bash[77063]: cluster 2023-10-09T04:38:52.836128+0000 mon.a (mon.0) 1081 : cluster 3 Health check failed: Degraded data redundancy: 5803/276867 objects degraded (2.096%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:38:53.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:53.362 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:53.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 15 94113 94098 76.1568 47.6875 0.0159771 0.0131251 2023-10-09T04:38:53.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:53.491 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:53.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:53.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:53.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:53.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:53.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:53.811 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:53.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:53.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:53.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:54.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.057 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:53 smithi082 bash[77063]: cluster 2023-10-09T04:38:52.461590+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 5803/276867 objects degraded (2.096%); 5755/276867 objects misplaced (2.079%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:54.058 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:53 smithi082 bash[79733]: cluster 2023-10-09T04:38:52.461590+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 5803/276867 objects degraded (2.096%); 5755/276867 objects misplaced (2.079%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:54.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:54.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:53 smithi153 bash[59565]: cluster 2023-10-09T04:38:52.461590+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v743: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 5803/276867 objects degraded (2.096%); 5755/276867 objects misplaced (2.079%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:54.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:54.340 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:54.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:54.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 94780 94764 75.7153 41.625 0.020376 0.0132007 2023-10-09T04:38:54.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:54.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:54.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:54.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:54.786 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:38:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:54.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:54.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:54.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:54.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:55.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:38:55.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:55.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:55.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:55.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:55.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:55.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 95376 95360 75.2297 37.25 0.00901314 0.0132863 2023-10-09T04:38:55.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:55.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:55.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:55.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:55.803 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:55.907 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: cluster 2023-10-09T04:38:54.462069+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v744: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 11551/279768 objects degraded (4.129%); 5755/279768 objects misplaced (2.057%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:55.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: audit 2023-10-09T04:38:55.060834+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:55.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: audit 2023-10-09T04:38:55.072804+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:55.908 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: audit 2023-10-09T04:38:55.759140+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:55.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:55.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:56.044 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:56.066 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:56.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: cluster 2023-10-09T04:38:54.462069+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v744: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 11551/279768 objects degraded (4.129%); 5755/279768 objects misplaced (2.057%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: audit 2023-10-09T04:38:55.060834+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: audit 2023-10-09T04:38:55.072804+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: audit 2023-10-09T04:38:55.759140+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:56.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: audit 2023-10-09T04:38:55.760379+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:56.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:55 smithi153 bash[59565]: audit 2023-10-09T04:38:55.767628+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: audit 2023-10-09T04:38:55.760379+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:56.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[77063]: audit 2023-10-09T04:38:55.767628+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: cluster 2023-10-09T04:38:54.462069+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v744: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 11551/279768 objects degraded (4.129%); 5755/279768 objects misplaced (2.057%); 149 KiB/s, 2 objects/s recovering 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: audit 2023-10-09T04:38:55.060834+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: audit 2023-10-09T04:38:55.072804+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: audit 2023-10-09T04:38:55.759140+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: audit 2023-10-09T04:38:55.760379+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:38:56.179 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:55 smithi082 bash[79733]: audit 2023-10-09T04:38:55.767628+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:38:56.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:56.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:56.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:56.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:38:56.385900+0000 min lat: 0.00131815 max lat: 2.28434 avg lat: 0.0133584 2023-10-09T04:38:56.387 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:56.387 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 96056 96040 74.8216 42.5 0.0121326 0.0133584 2023-10-09T04:38:56.450 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:56.456 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:56.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:56.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:56.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:56.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:56.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:56.736 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:56.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:56.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:56.904 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:56.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:57.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:57.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:57.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:57.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:57.274 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:57.302 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:57.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:57.386 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 96778 96762 74.4559 45.125 0.0112226 0.0134225 2023-10-09T04:38:57.406 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:57.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:57.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:57.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:57.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:57.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:57.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:57.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:57.913 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:57 smithi153 bash[59565]: cluster 2023-10-09T04:38:56.462745+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 11551/280830 objects degraded (4.113%); 5755/280830 objects misplaced (2.049%); 117 KiB/s, 1 objects/s recovering 2023-10-09T04:38:57.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:57.980 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:57.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:58.001 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:38:58.001 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:38:58.001 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-10-09T04:38:58.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:38:58.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:58.092 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:57 smithi082 bash[77063]: cluster 2023-10-09T04:38:56.462745+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 11551/280830 objects degraded (4.113%); 5755/280830 objects misplaced (2.049%); 117 KiB/s, 1 objects/s recovering 2023-10-09T04:38:58.096 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:57 smithi082 bash[79733]: cluster 2023-10-09T04:38:56.462745+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s; 11551/280830 objects degraded (4.113%); 5755/280830 objects misplaced (2.049%); 117 KiB/s, 1 objects/s recovering 2023-10-09T04:38:58.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:58.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:58.245 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:58.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:58.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:58.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:58.388 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 97217 97201 73.8823 27.4375 0.0326231 0.0135285 2023-10-09T04:38:58.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:58.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:58.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:58.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:58.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:58.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:58.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:58.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:58.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:58.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:58.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:59.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:59.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:59.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:59.067 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:58 smithi082 bash[77063]: cluster 2023-10-09T04:38:58.465597+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 11551/288438 objects degraded (4.005%); 5683/288438 objects misplaced (1.970%); 625 KiB/s, 9 objects/s recovering 2023-10-09T04:38:59.067 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:38:58 smithi082 bash[77063]: cluster 2023-10-09T04:38:58.910518+0000 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 11551/288438 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:38:59.068 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:58 smithi082 bash[79733]: cluster 2023-10-09T04:38:58.465597+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 11551/288438 objects degraded (4.005%); 5683/288438 objects misplaced (1.970%); 625 KiB/s, 9 objects/s recovering 2023-10-09T04:38:59.068 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:38:58 smithi082 bash[79733]: cluster 2023-10-09T04:38:58.910518+0000 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 11551/288438 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:38:59.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:59.158 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:58 smithi153 bash[59565]: cluster 2023-10-09T04:38:58.465597+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 11551/288438 objects degraded (4.005%); 5683/288438 objects misplaced (1.970%); 625 KiB/s, 9 objects/s recovering 2023-10-09T04:38:59.159 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:38:58 smithi153 bash[59565]: cluster 2023-10-09T04:38:58.910518+0000 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 11551/288438 objects degraded (4.005%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:38:59.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:59.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:59.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:59.388 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 15 97924 97909 73.5262 44.25 0.0128914 0.0135945 2023-10-09T04:38:59.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:59.459 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:59.509 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:59.513 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:59.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:59.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:59.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:59.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:38:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:38:59] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T04:38:59.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:59.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:59.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:59.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:59.962 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:59.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:38:59.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:00.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:00.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:00.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:00.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:00.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:00.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:00.388 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 98657 98641 73.1963 45.75 0.0191186 0.0136542 2023-10-09T04:39:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:00 smithi153 bash[59565]: audit 2023-10-09T04:38:59.281178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:00.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:00 smithi153 bash[59565]: audit 2023-10-09T04:38:59.285591+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:00.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:00.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:00.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:00.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:00.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:00 smithi082 bash[79733]: audit 2023-10-09T04:38:59.281178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:00 smithi082 bash[79733]: audit 2023-10-09T04:38:59.285591+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:00 smithi082 bash[77063]: audit 2023-10-09T04:38:59.281178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:00 smithi082 bash[77063]: audit 2023-10-09T04:38:59.285591+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:00.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:00.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:00.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:00.906 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:00.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:01.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:01.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:01.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:01.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:01.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:01.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:01.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 99394 99378 72.8778 46.0625 0.00961655 0.0137152 2023-10-09T04:39:01.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:01.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:01 smithi153 bash[59565]: cluster 2023-10-09T04:39:00.466307+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v747: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 11551/290862 objects degraded (3.971%); 5683/290862 objects misplaced (1.954%); 550 KiB/s, 8 objects/s recovering 2023-10-09T04:39:01.431 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:01 smithi082 bash[77063]: cluster 2023-10-09T04:39:00.466307+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v747: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 11551/290862 objects degraded (3.971%); 5683/290862 objects misplaced (1.954%); 550 KiB/s, 8 objects/s recovering 2023-10-09T04:39:01.431 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:01 smithi082 bash[79733]: cluster 2023-10-09T04:39:00.466307+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v747: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 11551/290862 objects degraded (3.971%); 5683/290862 objects misplaced (1.954%); 550 KiB/s, 8 objects/s recovering 2023-10-09T04:39:01.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:01.566 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:01.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:01.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:01.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:01.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:01.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:02.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:02.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 100109 100093 72.5508 44.6875 0.0105874 0.0137756 2023-10-09T04:39:02.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.422 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.527 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:02.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:02.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:02.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:02.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:02.931 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:02.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:02.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:03.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:03.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:03.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:03.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:03.325 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:03.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:03.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 100771 100755 72.1933 41.375 0.0209982 0.0138434 2023-10-09T04:39:03.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:03.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:03.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:03.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:03.601 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:03 smithi153 bash[59565]: cluster 2023-10-09T04:39:02.466925+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v748: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/295971 objects degraded (3.903%); 5621/295971 objects misplaced (1.899%); 803 KiB/s, 12 objects/s recovering 2023-10-09T04:39:03.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:03.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:03.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:03 smithi082 bash[77063]: cluster 2023-10-09T04:39:02.466925+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v748: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/295971 objects degraded (3.903%); 5621/295971 objects misplaced (1.899%); 803 KiB/s, 12 objects/s recovering 2023-10-09T04:39:03.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:03 smithi082 bash[79733]: cluster 2023-10-09T04:39:02.466925+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v748: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/295971 objects degraded (3.903%); 5621/295971 objects misplaced (1.899%); 803 KiB/s, 12 objects/s recovering 2023-10-09T04:39:03.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:03.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:03.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:03.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:03.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:04.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:04.115 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:04.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:04.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:04.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:04.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 101462 101446 71.8645 43.1875 0.0124006 0.0139069 2023-10-09T04:39:04.415 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:04.520 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:04.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:04.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:04.678 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:04.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:04 smithi153 bash[59565]: cluster 2023-10-09T04:39:04.523557+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 11551/299832 objects degraded (3.852%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:04.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:04.937 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:04 smithi082 bash[79733]: cluster 2023-10-09T04:39:04.523557+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 11551/299832 objects degraded (3.852%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:04.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:04 smithi082 bash[77063]: cluster 2023-10-09T04:39:04.523557+0000 mon.a (mon.0) 1090 : cluster 3 Health check update: Degraded data redundancy: 11551/299832 objects degraded (3.852%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:04.958 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:04.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:04.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:05.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:05.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:05.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:05.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:05.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:05.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 16 102133 102117 71.529 41.9375 0.0297405 0.0139742 2023-10-09T04:39:05.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:05.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:05.507 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:39:05.508 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:39:05.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:05.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:05.692 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:05.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:05.724 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:05 smithi082 bash[77063]: cluster 2023-10-09T04:39:04.467542+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v749: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/299832 objects degraded (3.852%); 5621/299832 objects misplaced (1.875%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:05.725 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:05 smithi082 bash[79733]: cluster 2023-10-09T04:39:04.467542+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v749: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/299832 objects degraded (3.852%); 5621/299832 objects misplaced (1.875%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:05.826 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:05.836 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:05 smithi153 bash[59565]: cluster 2023-10-09T04:39:04.467542+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v749: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 11551/299832 objects degraded (3.852%); 5621/299832 objects misplaced (1.875%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:05.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:05.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:05.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:06.120 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:06.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:06.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:06.250 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:06.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:06.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 90.0208 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 102657 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 71.2731 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 31.2352 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 161.625 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 1140 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 499.774 2023-10-09T04:39:06.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 2586 2023-10-09T04:39:06.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 0 2023-10-09T04:39:06.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.014025 2023-10-09T04:39:06.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.0291576 2023-10-09T04:39:06.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 2.28434 2023-10-09T04:39:06.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00131815 2023-10-09T04:39:06.391 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.399 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:39:06.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:06.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.558 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:06.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:06.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:06.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:06.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:06.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:06.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:06.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:06.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:07.060 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:07.096 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:07.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:07.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:07.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:07.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:07.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:07.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:07.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:07.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:07.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:07.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:07 smithi153 bash[59565]: cluster 2023-10-09T04:39:06.468277+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v750: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 11551/301182 objects degraded (3.835%); 5621/301182 objects misplaced (1.866%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:07.952 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:07 smithi082 bash[77063]: cluster 2023-10-09T04:39:06.468277+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v750: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 11551/301182 objects degraded (3.835%); 5621/301182 objects misplaced (1.866%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:07.953 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:07 smithi082 bash[79733]: cluster 2023-10-09T04:39:06.468277+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v750: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 11551/301182 objects degraded (3.835%); 5621/301182 objects misplaced (1.866%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:39:07.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:08.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:08.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:08.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:08.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.319 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:08.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:08.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:08.577 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:08.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:08.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:08.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:08.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:08.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:39:08.954 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:08.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:08.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:09.047 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:09.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:09.113 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:09.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:09.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:09.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:09.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:09.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:09.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:09.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:09.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:09.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:09.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:09.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:09.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:09.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:09.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:09 smithi082 bash[77063]: cluster 2023-10-09T04:39:08.468906+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v751: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 773 op/s; 11551/305682 objects degraded (3.779%); 5531/305682 objects misplaced (1.809%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:09.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:09 smithi082 bash[77063]: cluster 2023-10-09T04:39:09.666229+0000 mon.a (mon.0) 1091 : cluster 3 Health check update: Degraded data redundancy: 11551/305682 objects degraded (3.779%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:09.891 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:09] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T04:39:09.892 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:09 smithi082 bash[79733]: cluster 2023-10-09T04:39:08.468906+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v751: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 773 op/s; 11551/305682 objects degraded (3.779%); 5531/305682 objects misplaced (1.809%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:09.892 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:09 smithi082 bash[79733]: cluster 2023-10-09T04:39:09.666229+0000 mon.a (mon.0) 1091 : cluster 3 Health check update: Degraded data redundancy: 11551/305682 objects degraded (3.779%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:09.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:09.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:09.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:10.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:10.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:10.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:10.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:09 smithi153 bash[59565]: cluster 2023-10-09T04:39:08.468906+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v751: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 773 op/s; 11551/305682 objects degraded (3.779%); 5531/305682 objects misplaced (1.809%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:10.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:09 smithi153 bash[59565]: cluster 2023-10-09T04:39:09.666229+0000 mon.a (mon.0) 1091 : cluster 3 Health check update: Degraded data redundancy: 11551/305682 objects degraded (3.779%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:10.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:10.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:10.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:10.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:10.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:10.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:10.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:10.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:10.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:10.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:10.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:10.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:11.066 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:11.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:11.195 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:11.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:11.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:11.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:11.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:11.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:11.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:11.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:11.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:11.694 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:11 smithi082 bash[77063]: cluster 2023-10-09T04:39:10.469609+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v752: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 629 op/s; 11551/305004 objects degraded (3.787%); 5531/305004 objects misplaced (1.813%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:11.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:11.788 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:11.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:11.901 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:11 smithi082 bash[79733]: cluster 2023-10-09T04:39:10.469609+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v752: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 629 op/s; 11551/305004 objects degraded (3.787%); 5531/305004 objects misplaced (1.813%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:11.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:11.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:12.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:12.111 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:39:12.170 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:39:12.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:11 smithi153 bash[59565]: cluster 2023-10-09T04:39:10.469609+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v752: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 629 op/s; 11551/305004 objects degraded (3.787%); 5531/305004 objects misplaced (1.813%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:12.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:12.222 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:39:12.267 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:39:12.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.318 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:39:12.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:12.354 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:39:12.390 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:39:12.422 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.431 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:39:12.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:12.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:12.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:12.706 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:12.721 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:39:12.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:12.840 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.841 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[77063]: audit 2023-10-09T04:39:11.944562+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.82:0/883806107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:12.841 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[77063]: audit 2023-10-09T04:39:12.379389+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.82:0/2251288893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:12.841 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[77063]: audit 2023-10-09T04:39:12.379909+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:12.841 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[79733]: audit 2023-10-09T04:39:11.944562+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.82:0/883806107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:12.841 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[79733]: audit 2023-10-09T04:39:12.379389+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.82:0/2251288893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:12.842 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:12 smithi082 bash[79733]: audit 2023-10-09T04:39:12.379909+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:12.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:12.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:12.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:12 smithi153 bash[59565]: audit 2023-10-09T04:39:11.944562+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.82:0/883806107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:12.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:12 smithi153 bash[59565]: audit 2023-10-09T04:39:12.379389+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.82:0/2251288893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:12.983 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:12 smithi153 bash[59565]: audit 2023-10-09T04:39:12.379909+0000 mon.a (mon.0) 1092 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:13.004 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:13.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:13.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:13.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:13.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:13.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:13.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:13.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:13.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:13.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:13.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.760 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: cluster 2023-10-09T04:39:12.470231+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v753: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 712 op/s; 11551/299595 objects degraded (3.856%); 5448/299595 objects misplaced (1.818%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:13.761 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: cluster 2023-10-09T04:39:12.692556+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:13.761 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: cluster 2023-10-09T04:39:12.706730+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:13.761 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: cluster 2023-10-09T04:39:12.708607+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:13.761 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: audit 2023-10-09T04:39:12.710757+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:13.761 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:13 smithi153 bash[59565]: cluster 2023-10-09T04:39:12.714799+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:39:13.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:13.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:13.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:14.016 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:14.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:14.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:14.092 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: cluster 2023-10-09T04:39:12.470231+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v753: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 712 op/s; 11551/299595 objects degraded (3.856%); 5448/299595 objects misplaced (1.818%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: cluster 2023-10-09T04:39:12.692556+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: cluster 2023-10-09T04:39:12.706730+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: cluster 2023-10-09T04:39:12.708607+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: audit 2023-10-09T04:39:12.710757+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[77063]: cluster 2023-10-09T04:39:12.714799+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: cluster 2023-10-09T04:39:12.470231+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v753: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 712 op/s; 11551/299595 objects degraded (3.856%); 5448/299595 objects misplaced (1.818%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: cluster 2023-10-09T04:39:12.692556+0000 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:14.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: cluster 2023-10-09T04:39:12.706730+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:14.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: cluster 2023-10-09T04:39:12.708607+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T04:39:14.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: audit 2023-10-09T04:39:12.710757+0000 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:14.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:13 smithi082 bash[79733]: cluster 2023-10-09T04:39:12.714799+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:39:14.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:14.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:14.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:14.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:14.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:14.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:14.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:14.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:14.675 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:14.716 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:14.717 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:14 smithi082 bash[77063]: audit 2023-10-09T04:39:14.276907+0000 mon.a (mon.0) 1096 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:14 smithi082 bash[77063]: cluster 2023-10-09T04:39:14.670368+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 11551/299595 objects degraded (3.856%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:14.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:14.830 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:14 smithi153 bash[59565]: audit 2023-10-09T04:39:14.276907+0000 mon.a (mon.0) 1096 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.830 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:14 smithi153 bash[59565]: cluster 2023-10-09T04:39:14.670368+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 11551/299595 objects degraded (3.856%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:14.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:14.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:14.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:14 smithi082 bash[79733]: audit 2023-10-09T04:39:14.276907+0000 mon.a (mon.0) 1096 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:14 smithi082 bash[79733]: cluster 2023-10-09T04:39:14.670368+0000 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 11551/299595 objects degraded (3.856%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:14.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:14.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:15.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:15.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:15.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:15.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:15.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:15.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:15.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:15.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:15.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:15.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:15.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:15.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:15.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:15.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:15.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:15.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:15.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:15 smithi082 bash[77063]: cluster 2023-10-09T04:39:14.470833+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v755: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 681 op/s; 11551/295848 objects degraded (3.904%); 5448/295848 objects misplaced (1.841%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:15.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:15 smithi082 bash[79733]: cluster 2023-10-09T04:39:14.470833+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v755: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 681 op/s; 11551/295848 objects degraded (3.904%); 5448/295848 objects misplaced (1.841%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:16.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:16.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:16.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:16.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:15 smithi153 bash[59565]: cluster 2023-10-09T04:39:14.470833+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v755: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 681 op/s; 11551/295848 objects degraded (3.904%); 5448/295848 objects misplaced (1.841%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:16.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:16.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:16.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:16.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:16.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:16.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:16.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:16.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:16.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:16.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:16.794 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:16.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.862 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:16.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:16.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:16.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:16.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:17.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:17.125 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:17.230 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:17.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:17.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:17.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:17.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:17.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:17.528 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:39:17.535 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:39:17.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:17.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:17.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:17.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:17 smithi082 bash[77063]: cluster 2023-10-09T04:39:16.471582+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v756: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 676 op/s; 11551/294639 objects degraded (3.920%); 5448/294639 objects misplaced (1.849%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:17.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:17.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:17.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:17.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:17.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:17 smithi153 bash[59565]: cluster 2023-10-09T04:39:16.471582+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v756: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 676 op/s; 11551/294639 objects degraded (3.920%); 5448/294639 objects misplaced (1.849%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:17.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:17.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:17 smithi082 bash[79733]: cluster 2023-10-09T04:39:16.471582+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v756: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 676 op/s; 11551/294639 objects degraded (3.920%); 5448/294639 objects misplaced (1.849%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:39:18.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:18.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:18.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:18.208 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:18.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:39:18.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:39:18.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:18.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:18.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:18.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:18.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:18.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:18.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:18.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:18.887 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:18.898 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:18 smithi153 bash[59565]: audit 2023-10-09T04:39:17.503508+0000 mgr.y (mgr.34107) 754 : audit 0 from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:39:18.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:18.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:18.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:18 smithi082 bash[77063]: audit 2023-10-09T04:39:17.503508+0000 mgr.y (mgr.34107) 754 : audit 0 from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:39:18.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:18 smithi082 bash[79733]: audit 2023-10-09T04:39:17.503508+0000 mgr.y (mgr.34107) 754 : audit 0 from='client.45248 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:39:19.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:19.055 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:19.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:19.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:19.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:19.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:19.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:19.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:19.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:19.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:19.687 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:19.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:19.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:19.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.963 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:19 smithi153 bash[59565]: cluster 2023-10-09T04:39:18.472212+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.8 MiB/s wr, 683 op/s; 11639/286941 objects degraded (4.056%); 5292/286941 objects misplaced (1.844%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:19.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:19 smithi153 bash[59565]: cluster 2023-10-09T04:39:19.678096+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 11639/286941 objects degraded (4.056%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:19.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:19.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:19.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:20.017 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:19 smithi082 bash[77063]: cluster 2023-10-09T04:39:18.472212+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.8 MiB/s wr, 683 op/s; 11639/286941 objects degraded (4.056%); 5292/286941 objects misplaced (1.844%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:20.017 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:19 smithi082 bash[77063]: cluster 2023-10-09T04:39:19.678096+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 11639/286941 objects degraded (4.056%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:20.017 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:19] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-10-09T04:39:20.017 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:19 smithi082 bash[79733]: cluster 2023-10-09T04:39:18.472212+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.8 MiB/s wr, 683 op/s; 11639/286941 objects degraded (4.056%); 5292/286941 objects misplaced (1.844%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:20.018 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:19 smithi082 bash[79733]: cluster 2023-10-09T04:39:19.678096+0000 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 11639/286941 objects degraded (4.056%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:20.086 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:20.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:20.214 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":756,"stamp":"2023-10-09T04:39:16.471143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6439376843,"num_objects":98213,"num_object_clones":0,"num_object_copies":294639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11551,"num_objects_misplaced":5448,"num_objects_unfound":0,"num_objects_dirty":98213,"num_whiteouts":0,"num_read":6302,"num_read_kb":12736,"num_write":108237,"num_write_kb":6585208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":324,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108301,"ondisk_log_size":108301,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21581632,"kb_used_data":18924608,"kb_used_omap":46,"kb_used_meta":2656721,"kb_avail":728379584,"statfs":{"total":767960285184,"available":745860694016,"internally_reserved":0,"allocated":19378798592,"data_stored":19373547825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47544,"internal_metadata":2720482888},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,5,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-142999528,"num_objects":-2181,"num_object_clones":0,"num_object_copies":-6543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-173,"num_objects_unfound":0,"num_objects_dirty":-2181,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6765,"num_write_kb":146625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":173,"num_bytes_recovered":11337728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003396"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":164,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.713998+0000","last_change":"2023-10-09T04:23:18.769417+0000","last_active":"2023-10-09T04:38:50.713998+0000","last_peered":"2023-10-09T04:38:50.713998+0000","last_clean":"2023-10-09T04:38:50.713998+0000","last_became_active":"2023-10-09T04:23:18.769156+0000","last_became_peered":"2023-10-09T04:23:18.769156+0000","last_unstale":"2023-10-09T04:38:50.713998+0000","last_undegraded":"2023-10-09T04:38:50.713998+0000","last_fullsized":"2023-10-09T04:38:50.713998+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.670686+0000","last_change":"2023-10-09T04:28:51.530770+0000","last_active":"2023-10-09T04:38:50.670686+0000","last_peered":"2023-10-09T04:38:50.670686+0000","last_clean":"2023-10-09T04:38:50.670686+0000","last_became_active":"2023-10-09T04:28:51.530387+0000","last_became_peered":"2023-10-09T04:28:51.530387+0000","last_unstale":"2023-10-09T04:38:50.670686+0000","last_undegraded":"2023-10-09T04:38:50.670686+0000","last_fullsized":"2023-10-09T04:38:50.670686+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:16.201179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.845196+0000","last_change":"2023-10-09T04:38:49.687475+0000","last_active":"2023-10-09T04:38:50.845196+0000","last_peered":"2023-10-09T04:38:50.845196+0000","last_clean":"2023-10-09T04:38:50.845196+0000","last_became_active":"2023-10-09T04:38:49.687077+0000","last_became_peered":"2023-10-09T04:38:49.687077+0000","last_unstale":"2023-10-09T04:38:50.845196+0000","last_undegraded":"2023-10-09T04:38:50.845196+0000","last_fullsized":"2023-10-09T04:38:50.845196+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:41.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671959+0000","last_change":"2023-10-09T04:24:00.822517+0000","last_active":"2023-10-09T04:38:49.671959+0000","last_peered":"2023-10-09T04:38:49.671959+0000","last_clean":"2023-10-09T04:38:49.671959+0000","last_became_active":"2023-10-09T04:24:00.821940+0000","last_became_peered":"2023-10-09T04:24:00.821940+0000","last_unstale":"2023-10-09T04:38:49.671959+0000","last_undegraded":"2023-10-09T04:38:49.671959+0000","last_fullsized":"2023-10-09T04:38:49.671959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:26.704065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.670108+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:38:49.670108+0000","last_peered":"2023-10-09T04:38:49.670108+0000","last_clean":"2023-10-09T04:38:49.670108+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:38:49.670108+0000","last_undegraded":"2023-10-09T04:38:49.670108+0000","last_fullsized":"2023-10-09T04:38:49.670108+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":139,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676958+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:38:50.676958+0000","last_peered":"2023-10-09T04:38:50.676958+0000","last_clean":"2023-10-09T04:38:50.676958+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:38:50.676958+0000","last_undegraded":"2023-10-09T04:38:50.676958+0000","last_fullsized":"2023-10-09T04:38:50.676958+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671844+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:38:49.671844+0000","last_peered":"2023-10-09T04:38:49.671844+0000","last_clean":"2023-10-09T04:38:49.671844+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:38:49.671844+0000","last_undegraded":"2023-10-09T04:38:49.671844+0000","last_fullsized":"2023-10-09T04:38:49.671844+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":160,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711371+0000","last_change":"2023-10-09T04:34:09.918534+0000","last_active":"2023-10-09T04:38:50.711371+0000","last_peered":"2023-10-09T04:38:50.711371+0000","last_clean":"2023-10-09T04:38:50.711371+0000","last_became_active":"2023-10-09T04:34:09.918013+0000","last_became_peered":"2023-10-09T04:34:09.918013+0000","last_unstale":"2023-10-09T04:38:50.711371+0000","last_undegraded":"2023-10-09T04:38:50.711371+0000","last_fullsized":"2023-10-09T04:38:50.711371+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:00.542758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672367+0000","last_change":"2023-10-09T04:34:36.648444+0000","last_active":"2023-10-09T04:38:50.672367+0000","last_peered":"2023-10-09T04:38:50.672367+0000","last_clean":"2023-10-09T04:38:50.672367+0000","last_became_active":"2023-10-09T04:34:36.647994+0000","last_became_peered":"2023-10-09T04:34:36.647994+0000","last_unstale":"2023-10-09T04:38:50.672367+0000","last_undegraded":"2023-10-09T04:38:50.672367+0000","last_fullsized":"2023-10-09T04:38:50.672367+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.049665+0000","last_change":"2023-10-09T04:34:09.912750+0000","last_active":"2023-10-09T04:38:51.049665+0000","last_peered":"2023-10-09T04:38:51.049665+0000","last_clean":"2023-10-09T04:38:51.049665+0000","last_became_active":"2023-10-09T04:34:09.908040+0000","last_became_peered":"2023-10-09T04:34:09.908040+0000","last_unstale":"2023-10-09T04:38:51.049665+0000","last_undegraded":"2023-10-09T04:38:51.049665+0000","last_fullsized":"2023-10-09T04:38:51.049665+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":152,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.882236+0000","last_change":"2023-10-09T04:24:00.824188+0000","last_active":"2023-10-09T04:38:50.882236+0000","last_peered":"2023-10-09T04:38:50.882236+0000","last_clean":"2023-10-09T04:38:50.882236+0000","last_became_active":"2023-10-09T04:24:00.823535+0000","last_became_peered":"2023-10-09T04:24:00.823535+0000","last_unstale":"2023-10-09T04:38:50.882236+0000","last_undegraded":"2023-10-09T04:38:50.882236+0000","last_fullsized":"2023-10-09T04:38:50.882236+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:08.790968+0000","last_change":"2023-10-09T04:34:10.030630+0000","last_active":"2023-10-09T04:39:08.790968+0000","last_peered":"2023-10-09T04:39:08.790968+0000","last_clean":"2023-10-09T04:39:08.790968+0000","last_became_active":"2023-10-09T04:34:10.030403+0000","last_became_peered":"2023-10-09T04:34:10.030403+0000","last_unstale":"2023-10-09T04:39:08.790968+0000","last_undegraded":"2023-10-09T04:39:08.790968+0000","last_fullsized":"2023-10-09T04:39:08.790968+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:53.085670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":170,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.724980+0000","last_change":"2023-10-09T04:34:09.968067+0000","last_active":"2023-10-09T04:38:50.724980+0000","last_peered":"2023-10-09T04:38:50.724980+0000","last_clean":"2023-10-09T04:38:50.724980+0000","last_became_active":"2023-10-09T04:34:09.967671+0000","last_became_peered":"2023-10-09T04:34:09.967671+0000","last_unstale":"2023-10-09T04:38:50.724980+0000","last_undegraded":"2023-10-09T04:38:50.724980+0000","last_fullsized":"2023-10-09T04:38:50.724980+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":162,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.714736+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:38:50.714736+0000","last_peered":"2023-10-09T04:38:50.714736+0000","last_clean":"2023-10-09T04:38:50.714736+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:38:50.714736+0000","last_undegraded":"2023-10-09T04:38:50.714736+0000","last_fullsized":"2023-10-09T04:38:50.714736+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":151,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881277+0000","last_change":"2023-10-09T04:23:18.770172+0000","last_active":"2023-10-09T04:38:50.881277+0000","last_peered":"2023-10-09T04:38:50.881277+0000","last_clean":"2023-10-09T04:38:50.881277+0000","last_became_active":"2023-10-09T04:23:18.769750+0000","last_became_peered":"2023-10-09T04:23:18.769750+0000","last_unstale":"2023-10-09T04:38:50.881277+0000","last_undegraded":"2023-10-09T04:38:50.881277+0000","last_fullsized":"2023-10-09T04:38:50.881277+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":145,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711435+0000","last_change":"2023-10-09T04:28:51.524085+0000","last_active":"2023-10-09T04:38:50.711435+0000","last_peered":"2023-10-09T04:38:50.711435+0000","last_clean":"2023-10-09T04:38:50.711435+0000","last_became_active":"2023-10-09T04:28:51.523467+0000","last_became_peered":"2023-10-09T04:28:51.523467+0000","last_unstale":"2023-10-09T04:38:50.711435+0000","last_undegraded":"2023-10-09T04:38:50.711435+0000","last_fullsized":"2023-10-09T04:38:50.711435+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:46.898864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":136,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671937+0000","last_change":"2023-10-09T04:24:00.826227+0000","last_active":"2023-10-09T04:38:49.671937+0000","last_peered":"2023-10-09T04:38:49.671937+0000","last_clean":"2023-10-09T04:38:49.671937+0000","last_became_active":"2023-10-09T04:24:00.825881+0000","last_became_peered":"2023-10-09T04:24:00.825881+0000","last_unstale":"2023-10-09T04:38:49.671937+0000","last_undegraded":"2023-10-09T04:38:49.671937+0000","last_fullsized":"2023-10-09T04:38:49.671937+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:58.017059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672375+0000","last_change":"2023-10-09T04:34:36.646566+0000","last_active":"2023-10-09T04:38:50.672375+0000","last_peered":"2023-10-09T04:38:50.672375+0000","last_clean":"2023-10-09T04:38:50.672375+0000","last_became_active":"2023-10-09T04:34:36.646266+0000","last_became_peered":"2023-10-09T04:34:36.646266+0000","last_unstale":"2023-10-09T04:38:50.672375+0000","last_undegraded":"2023-10-09T04:38:50.672375+0000","last_fullsized":"2023-10-09T04:38:50.672375+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":149,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.049766+0000","last_change":"2023-10-09T04:34:09.907681+0000","last_active":"2023-10-09T04:38:51.049766+0000","last_peered":"2023-10-09T04:38:51.049766+0000","last_clean":"2023-10-09T04:38:51.049766+0000","last_became_active":"2023-10-09T04:34:09.907156+0000","last_became_peered":"2023-10-09T04:34:09.907156+0000","last_unstale":"2023-10-09T04:38:51.049766+0000","last_undegraded":"2023-10-09T04:38:51.049766+0000","last_fullsized":"2023-10-09T04:38:51.049766+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.672080+0000","last_change":"2023-10-09T04:24:00.822273+0000","last_active":"2023-10-09T04:38:49.672080+0000","last_peered":"2023-10-09T04:38:49.672080+0000","last_clean":"2023-10-09T04:38:49.672080+0000","last_became_active":"2023-10-09T04:24:00.821466+0000","last_became_peered":"2023-10-09T04:24:00.821466+0000","last_unstale":"2023-10-09T04:38:49.672080+0000","last_undegraded":"2023-10-09T04:38:49.672080+0000","last_fullsized":"2023-10-09T04:38:49.672080+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:07.032327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":148,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676260+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:38:50.676260+0000","last_peered":"2023-10-09T04:38:50.676260+0000","last_clean":"2023-10-09T04:38:50.676260+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:38:50.676260+0000","last_undegraded":"2023-10-09T04:38:50.676260+0000","last_fullsized":"2023-10-09T04:38:50.676260+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":141,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881782+0000","last_change":"2023-10-09T04:28:51.526352+0000","last_active":"2023-10-09T04:38:50.881782+0000","last_peered":"2023-10-09T04:38:50.881782+0000","last_clean":"2023-10-09T04:38:50.881782+0000","last_became_active":"2023-10-09T04:28:51.525911+0000","last_became_peered":"2023-10-09T04:28:51.525911+0000","last_unstale":"2023-10-09T04:38:50.881782+0000","last_undegraded":"2023-10-09T04:38:50.881782+0000","last_fullsized":"2023-10-09T04:38:50.881782+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"183'6563","reported_seq":6581,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:13.788412+0000","last_change":"2023-10-09T04:37:13.779184+0000","last_active":"2023-10-09T04:39:13.788412+0000","last_peered":"2023-10-09T04:39:13.788412+0000","last_clean":"2023-10-09T04:39:13.788412+0000","last_became_active":"2023-10-09T04:37:13.778902+0000","last_became_peered":"2023-10-09T04:37:13.778902+0000","last_unstale":"2023-10-09T04:39:13.788412+0000","last_undegraded":"2023-10-09T04:39:13.788412+0000","last_fullsized":"2023-10-09T04:39:13.788412+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:15.517709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394592256,"num_objects":6021,"num_object_clones":0,"num_object_copies":18063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6563,"num_write_kb":402688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":150,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881928+0000","last_change":"2023-10-09T04:22:23.618075+0000","last_active":"2023-10-09T04:38:50.881928+0000","last_peered":"2023-10-09T04:38:50.881928+0000","last_clean":"2023-10-09T04:38:50.881928+0000","last_became_active":"2023-10-09T04:22:23.617721+0000","last_became_peered":"2023-10-09T04:22:23.617721+0000","last_unstale":"2023-10-09T04:38:50.881928+0000","last_undegraded":"2023-10-09T04:38:50.881928+0000","last_fullsized":"2023-10-09T04:38:50.881928+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672157+0000","last_change":"2023-10-09T04:34:36.641326+0000","last_active":"2023-10-09T04:38:50.672157+0000","last_peered":"2023-10-09T04:38:50.672157+0000","last_clean":"2023-10-09T04:38:50.672157+0000","last_became_active":"2023-10-09T04:34:36.639824+0000","last_became_peered":"2023-10-09T04:34:36.639824+0000","last_unstale":"2023-10-09T04:38:50.672157+0000","last_undegraded":"2023-10-09T04:38:50.672157+0000","last_fullsized":"2023-10-09T04:38:50.672157+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"182'6699","reported_seq":6716,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:11.814135+0000","last_change":"2023-10-09T04:37:13.765303+0000","last_active":"2023-10-09T04:39:11.814135+0000","last_peered":"2023-10-09T04:39:11.814135+0000","last_clean":"2023-10-09T04:39:11.814135+0000","last_became_active":"2023-10-09T04:37:13.764983+0000","last_became_peered":"2023-10-09T04:37:13.764983+0000","last_unstale":"2023-10-09T04:39:11.814135+0000","last_undegraded":"2023-10-09T04:39:11.814135+0000","last_fullsized":"2023-10-09T04:39:11.814135+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.571025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404422656,"num_objects":6171,"num_object_clones":0,"num_object_copies":18513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6699,"num_write_kb":411840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":190,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881587+0000","last_change":"2023-10-09T04:38:49.743251+0000","last_active":"2023-10-09T04:38:50.881587+0000","last_peered":"2023-10-09T04:38:50.881587+0000","last_clean":"2023-10-09T04:38:50.881587+0000","last_became_active":"2023-10-09T04:38:49.696329+0000","last_became_peered":"2023-10-09T04:38:49.696329+0000","last_unstale":"2023-10-09T04:38:50.881587+0000","last_undegraded":"2023-10-09T04:38:50.881587+0000","last_fullsized":"2023-10-09T04:38:50.881587+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":162,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.729008+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:38:50.729008+0000","last_peered":"2023-10-09T04:38:50.729008+0000","last_clean":"2023-10-09T04:38:50.729008+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:38:50.729008+0000","last_undegraded":"2023-10-09T04:38:50.729008+0000","last_fullsized":"2023-10-09T04:38:50.729008+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"182'6728","reported_seq":6754,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:39:12.005692+0000","last_change":"2023-10-09T04:38:51.083542+0000","last_active":"2023-10-09T04:39:12.005692+0000","last_peered":"2023-10-09T04:39:12.005692+0000","last_clean":"2023-10-09T04:38:48.795614+0000","last_became_active":"2023-10-09T04:38:51.061751+0000","last_became_peered":"2023-10-09T04:38:51.061751+0000","last_unstale":"2023-10-09T04:39:12.005692+0000","last_undegraded":"2023-10-09T04:38:50.706378+0000","last_fullsized":"2023-10-09T04:38:50.672339+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:43.454692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405798912,"num_objects":6192,"num_object_clones":0,"num_object_copies":18576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5803,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6728,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1","objects":389},{"shards":"0,1,4","objects":5535}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":161,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881928+0000","last_change":"2023-10-09T04:34:36.648398+0000","last_active":"2023-10-09T04:38:50.881928+0000","last_peered":"2023-10-09T04:38:50.881928+0000","last_clean":"2023-10-09T04:38:50.881928+0000","last_became_active":"2023-10-09T04:34:36.647896+0000","last_became_peered":"2023-10-09T04:34:36.647896+0000","last_unstale":"2023-10-09T04:38:50.881928+0000","last_undegraded":"2023-10-09T04:38:50.881928+0000","last_fullsized":"2023-10-09T04:38:50.881928+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676306+0000","last_change":"2023-10-09T04:34:09.983232+0000","last_active":"2023-10-09T04:38:50.676306+0000","last_peered":"2023-10-09T04:38:50.676306+0000","last_clean":"2023-10-09T04:38:50.676306+0000","last_became_active":"2023-10-09T04:34:09.976353+0000","last_became_peered":"2023-10-09T04:34:09.976353+0000","last_unstale":"2023-10-09T04:38:50.676306+0000","last_undegraded":"2023-10-09T04:38:50.676306+0000","last_fullsized":"2023-10-09T04:38:50.676306+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"182'6768","reported_seq":6786,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:11.780403+0000","last_change":"2023-10-09T04:37:13.774933+0000","last_active":"2023-10-09T04:39:11.780403+0000","last_peered":"2023-10-09T04:39:11.780403+0000","last_clean":"2023-10-09T04:39:11.780403+0000","last_became_active":"2023-10-09T04:37:13.774485+0000","last_became_peered":"2023-10-09T04:37:13.774485+0000","last_unstale":"2023-10-09T04:39:11.780403+0000","last_undegraded":"2023-10-09T04:39:11.780403+0000","last_fullsized":"2023-10-09T04:39:11.780403+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6768,"log_dups_size":0,"ondisk_log_size":6768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.541922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411697152,"num_objects":6282,"num_object_clones":0,"num_object_copies":18846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6768,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672214+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:38:50.672214+0000","last_peered":"2023-10-09T04:38:50.672214+0000","last_clean":"2023-10-09T04:38:50.672214+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:38:50.672214+0000","last_undegraded":"2023-10-09T04:38:50.672214+0000","last_fullsized":"2023-10-09T04:38:50.672214+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":149,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671321+0000","last_change":"2023-10-09T04:23:18.763791+0000","last_active":"2023-10-09T04:38:49.671321+0000","last_peered":"2023-10-09T04:38:49.671321+0000","last_clean":"2023-10-09T04:38:49.671321+0000","last_became_active":"2023-10-09T04:23:18.763582+0000","last_became_peered":"2023-10-09T04:23:18.763582+0000","last_unstale":"2023-10-09T04:38:49.671321+0000","last_undegraded":"2023-10-09T04:38:49.671321+0000","last_fullsized":"2023-10-09T04:38:49.671321+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:11.433840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"182'6640","reported_seq":6658,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:12.007344+0000","last_change":"2023-10-09T04:37:13.769259+0000","last_active":"2023-10-09T04:39:12.007344+0000","last_peered":"2023-10-09T04:39:12.007344+0000","last_clean":"2023-10-09T04:39:12.007344+0000","last_became_active":"2023-10-09T04:37:13.768831+0000","last_became_peered":"2023-10-09T04:37:13.768831+0000","last_unstale":"2023-10-09T04:39:12.007344+0000","last_undegraded":"2023-10-09T04:39:12.007344+0000","last_fullsized":"2023-10-09T04:39:12.007344+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6640,"log_dups_size":0,"ondisk_log_size":6640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:41.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401735680,"num_objects":6130,"num_object_clones":0,"num_object_copies":18390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":99,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671452+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:38:49.671452+0000","last_peered":"2023-10-09T04:38:49.671452+0000","last_clean":"2023-10-09T04:38:49.671452+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:38:49.671452+0000","last_undegraded":"2023-10-09T04:38:49.671452+0000","last_fullsized":"2023-10-09T04:38:49.671452+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":162,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.725078+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:38:50.725078+0000","last_peered":"2023-10-09T04:38:50.725078+0000","last_clean":"2023-10-09T04:38:50.725078+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:38:50.725078+0000","last_undegraded":"2023-10-09T04:38:50.725078+0000","last_fullsized":"2023-10-09T04:38:50.725078+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"182'6664","reported_seq":6682,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:11.813046+0000","last_change":"2023-10-09T04:37:13.764469+0000","last_active":"2023-10-09T04:39:11.813046+0000","last_peered":"2023-10-09T04:39:11.813046+0000","last_clean":"2023-10-09T04:39:11.813046+0000","last_became_active":"2023-10-09T04:37:13.763760+0000","last_became_peered":"2023-10-09T04:37:13.763760+0000","last_unstale":"2023-10-09T04:39:11.813046+0000","last_undegraded":"2023-10-09T04:39:11.813046+0000","last_fullsized":"2023-10-09T04:39:11.813046+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6664,"log_dups_size":0,"ondisk_log_size":6664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:08.322588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404226048,"num_objects":6168,"num_object_clones":0,"num_object_copies":18504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6664,"num_write_kb":410624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":151,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676268+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:38:50.676268+0000","last_peered":"2023-10-09T04:38:50.676268+0000","last_clean":"2023-10-09T04:38:50.676268+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:38:50.676268+0000","last_undegraded":"2023-10-09T04:38:50.676268+0000","last_fullsized":"2023-10-09T04:38:50.676268+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":152,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.670483+0000","last_change":"2023-10-09T04:38:49.688633+0000","last_active":"2023-10-09T04:38:50.670483+0000","last_peered":"2023-10-09T04:38:50.670483+0000","last_clean":"2023-10-09T04:38:50.670483+0000","last_became_active":"2023-10-09T04:38:49.688312+0000","last_became_peered":"2023-10-09T04:38:49.688312+0000","last_unstale":"2023-10-09T04:38:50.670483+0000","last_undegraded":"2023-10-09T04:38:50.670483+0000","last_fullsized":"2023-10-09T04:38:50.670483+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:03.302885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"183'6622","reported_seq":6641,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:13.047723+0000","last_change":"2023-10-09T04:37:13.777270+0000","last_active":"2023-10-09T04:39:13.047723+0000","last_peered":"2023-10-09T04:39:13.047723+0000","last_clean":"2023-10-09T04:39:13.047723+0000","last_became_active":"2023-10-09T04:37:13.776717+0000","last_became_peered":"2023-10-09T04:37:13.776717+0000","last_unstale":"2023-10-09T04:39:13.047723+0000","last_undegraded":"2023-10-09T04:39:13.047723+0000","last_fullsized":"2023-10-09T04:39:13.047723+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6622,"log_dups_size":0,"ondisk_log_size":6622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397738008,"num_objects":6070,"num_object_clones":0,"num_object_copies":18210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6070,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6622,"num_write_kb":406081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.669773+0000","last_change":"2023-10-09T04:38:49.680323+0000","last_active":"2023-10-09T04:38:50.669773+0000","last_peered":"2023-10-09T04:38:50.669773+0000","last_clean":"2023-10-09T04:38:50.669773+0000","last_became_active":"2023-10-09T04:38:49.679872+0000","last_became_peered":"2023-10-09T04:38:49.679872+0000","last_unstale":"2023-10-09T04:38:50.669773+0000","last_undegraded":"2023-10-09T04:38:50.669773+0000","last_fullsized":"2023-10-09T04:38:50.669773+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:22.514987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":162,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.714497+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:38:50.714497+0000","last_peered":"2023-10-09T04:38:50.714497+0000","last_clean":"2023-10-09T04:38:50.714497+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:38:50.714497+0000","last_undegraded":"2023-10-09T04:38:50.714497+0000","last_fullsized":"2023-10-09T04:38:50.714497+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"183'6744","reported_seq":6762,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:13.073135+0000","last_change":"2023-10-09T04:37:13.778215+0000","last_active":"2023-10-09T04:39:13.073135+0000","last_peered":"2023-10-09T04:39:13.073135+0000","last_clean":"2023-10-09T04:39:13.073135+0000","last_became_active":"2023-10-09T04:37:13.777797+0000","last_became_peered":"2023-10-09T04:37:13.777797+0000","last_unstale":"2023-10-09T04:39:13.073135+0000","last_undegraded":"2023-10-09T04:39:13.073135+0000","last_fullsized":"2023-10-09T04:39:13.073135+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403177472,"num_objects":6152,"num_object_clones":0,"num_object_copies":18456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6744,"num_write_kb":412672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711144+0000","last_change":"2023-10-09T04:28:51.524392+0000","last_active":"2023-10-09T04:38:50.711144+0000","last_peered":"2023-10-09T04:38:50.711144+0000","last_clean":"2023-10-09T04:38:50.711144+0000","last_became_active":"2023-10-09T04:28:51.523517+0000","last_became_peered":"2023-10-09T04:28:51.523517+0000","last_unstale":"2023-10-09T04:38:50.711144+0000","last_undegraded":"2023-10-09T04:38:50.711144+0000","last_fullsized":"2023-10-09T04:38:50.711144+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:57.184425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711802+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:38:50.711802+0000","last_peered":"2023-10-09T04:38:50.711802+0000","last_clean":"2023-10-09T04:38:50.711802+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:38:50.711802+0000","last_undegraded":"2023-10-09T04:38:50.711802+0000","last_fullsized":"2023-10-09T04:38:50.711802+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"183'6658","reported_seq":6683,"reported_epoch":183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:39:13.028547+0000","last_change":"2023-10-09T04:38:50.861935+0000","last_active":"2023-10-09T04:39:13.028547+0000","last_peered":"2023-10-09T04:39:13.028547+0000","last_clean":"2023-10-09T04:38:48.801246+0000","last_became_active":"2023-10-09T04:38:50.851077+0000","last_became_peered":"2023-10-09T04:38:50.851077+0000","last_unstale":"2023-10-09T04:39:13.028547+0000","last_undegraded":"2023-10-09T04:38:50.709088+0000","last_fullsized":"2023-10-09T04:38:50.676122+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6658,"log_dups_size":0,"ondisk_log_size":6658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396099584,"num_objects":6044,"num_object_clones":0,"num_object_copies":18132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5748,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6658,"num_write_kb":406464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":5441},{"shards":"3,6","objects":296}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":558,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711780+0000","last_change":"2023-10-09T04:22:50.694870+0000","last_active":"2023-10-09T04:38:50.711780+0000","last_peered":"2023-10-09T04:38:50.711780+0000","last_clean":"2023-10-09T04:38:50.711780+0000","last_became_active":"2023-10-09T04:22:50.694129+0000","last_became_peered":"2023-10-09T04:22:50.694129+0000","last_unstale":"2023-10-09T04:38:50.711780+0000","last_undegraded":"2023-10-09T04:38:50.711780+0000","last_fullsized":"2023-10-09T04:38:50.711780+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:57.334016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672552+0000","last_change":"2023-10-09T04:34:36.645924+0000","last_active":"2023-10-09T04:38:50.672552+0000","last_peered":"2023-10-09T04:38:50.672552+0000","last_clean":"2023-10-09T04:38:50.672552+0000","last_became_active":"2023-10-09T04:34:36.645572+0000","last_became_peered":"2023-10-09T04:34:36.645572+0000","last_unstale":"2023-10-09T04:38:50.672552+0000","last_undegraded":"2023-10-09T04:38:50.672552+0000","last_fullsized":"2023-10-09T04:38:50.672552+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711392+0000","last_change":"2023-10-09T04:38:49.685222+0000","last_active":"2023-10-09T04:38:50.711392+0000","last_peered":"2023-10-09T04:38:50.711392+0000","last_clean":"2023-10-09T04:38:50.711392+0000","last_became_active":"2023-10-09T04:38:49.684705+0000","last_became_peered":"2023-10-09T04:38:49.684705+0000","last_unstale":"2023-10-09T04:38:50.711392+0000","last_undegraded":"2023-10-09T04:38:50.711392+0000","last_fullsized":"2023-10-09T04:38:50.711392+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:42.039283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"183'6744","reported_seq":6762,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:13.030205+0000","last_change":"2023-10-09T04:37:13.778105+0000","last_active":"2023-10-09T04:39:13.030205+0000","last_peered":"2023-10-09T04:39:13.030205+0000","last_clean":"2023-10-09T04:39:13.030205+0000","last_became_active":"2023-10-09T04:37:13.777582+0000","last_became_peered":"2023-10-09T04:37:13.777582+0000","last_unstale":"2023-10-09T04:39:13.030205+0000","last_undegraded":"2023-10-09T04:39:13.030205+0000","last_fullsized":"2023-10-09T04:39:13.030205+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404750336,"num_objects":6176,"num_object_clones":0,"num_object_copies":18528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6744,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":645,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672155+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:38:50.672155+0000","last_peered":"2023-10-09T04:38:50.672155+0000","last_clean":"2023-10-09T04:38:50.672155+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:38:50.672155+0000","last_undegraded":"2023-10-09T04:38:50.672155+0000","last_fullsized":"2023-10-09T04:38:50.672155+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.714693+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:38:50.714693+0000","last_peered":"2023-10-09T04:38:50.714693+0000","last_clean":"2023-10-09T04:38:50.714693+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:38:50.714693+0000","last_undegraded":"2023-10-09T04:38:50.714693+0000","last_fullsized":"2023-10-09T04:38:50.714693+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":148,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711548+0000","last_change":"2023-10-09T04:24:00.814479+0000","last_active":"2023-10-09T04:38:50.711548+0000","last_peered":"2023-10-09T04:38:50.711548+0000","last_clean":"2023-10-09T04:38:50.711548+0000","last_became_active":"2023-10-09T04:24:00.814099+0000","last_became_peered":"2023-10-09T04:24:00.814099+0000","last_unstale":"2023-10-09T04:38:50.711548+0000","last_undegraded":"2023-10-09T04:38:50.711548+0000","last_fullsized":"2023-10-09T04:38:50.711548+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.111972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"182'6690","reported_seq":7355,"reported_epoch":182,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:39:12.078057+0000","last_change":"2023-10-09T04:38:50.917315+0000","last_active":"2023-10-09T04:39:12.078057+0000","last_peered":"2023-10-09T04:39:12.078057+0000","last_clean":"2023-10-09T04:38:48.817443+0000","last_became_active":"2023-10-09T04:38:50.847982+0000","last_became_peered":"2023-10-09T04:38:50.847982+0000","last_unstale":"2023-10-09T04:39:12.078057+0000","last_undegraded":"2023-10-09T04:39:12.078057+0000","last_fullsized":"2023-10-09T04:38:50.672708+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:46.176442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405667840,"num_objects":6190,"num_object_clones":0,"num_object_copies":18570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5448,"num_objects_unfound":0,"num_objects_dirty":6190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6690,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":321,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1","objects":742},{"shards":"0,1,4","objects":5448}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1293,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.665200+0000","last_change":"2023-10-09T04:24:00.811230+0000","last_active":"2023-10-09T04:38:49.665200+0000","last_peered":"2023-10-09T04:38:49.665200+0000","last_clean":"2023-10-09T04:38:49.665200+0000","last_became_active":"2023-10-09T04:24:00.810614+0000","last_became_peered":"2023-10-09T04:24:00.810614+0000","last_unstale":"2023-10-09T04:38:49.665200+0000","last_undegraded":"2023-10-09T04:38:49.665200+0000","last_fullsized":"2023-10-09T04:38:49.665200+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:21.486587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:53.764760+0000","last_change":"2023-10-09T04:34:09.901059+0000","last_active":"2023-10-09T04:38:53.764760+0000","last_peered":"2023-10-09T04:38:53.764760+0000","last_clean":"2023-10-09T04:38:53.764760+0000","last_became_active":"2023-10-09T04:34:09.900690+0000","last_became_peered":"2023-10-09T04:34:09.900690+0000","last_unstale":"2023-10-09T04:38:53.764760+0000","last_undegraded":"2023-10-09T04:38:53.764760+0000","last_fullsized":"2023-10-09T04:38:53.764760+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:59.089033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":178,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711876+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:38:50.711876+0000","last_peered":"2023-10-09T04:38:50.711876+0000","last_clean":"2023-10-09T04:38:50.711876+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:38:50.711876+0000","last_undegraded":"2023-10-09T04:38:50.711876+0000","last_fullsized":"2023-10-09T04:38:50.711876+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"182'6593","reported_seq":6611,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:12.081590+0000","last_change":"2023-10-09T04:37:13.768144+0000","last_active":"2023-10-09T04:39:12.081590+0000","last_peered":"2023-10-09T04:39:12.081590+0000","last_clean":"2023-10-09T04:39:12.081590+0000","last_became_active":"2023-10-09T04:37:13.767811+0000","last_became_peered":"2023-10-09T04:37:13.767811+0000","last_unstale":"2023-10-09T04:39:12.081590+0000","last_undegraded":"2023-10-09T04:39:12.081590+0000","last_fullsized":"2023-10-09T04:39:12.081590+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:06.150093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6593,"num_write_kb":408128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":627,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672144+0000","last_change":"2023-10-09T04:34:36.645853+0000","last_active":"2023-10-09T04:38:50.672144+0000","last_peered":"2023-10-09T04:38:50.672144+0000","last_clean":"2023-10-09T04:38:50.672144+0000","last_became_active":"2023-10-09T04:34:36.644515+0000","last_became_peered":"2023-10-09T04:34:36.644515+0000","last_unstale":"2023-10-09T04:38:50.672144+0000","last_undegraded":"2023-10-09T04:38:50.672144+0000","last_fullsized":"2023-10-09T04:38:50.672144+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":400,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.884321+0000","last_change":"2023-10-09T04:34:09.917744+0000","last_active":"2023-10-09T04:38:50.884321+0000","last_peered":"2023-10-09T04:38:50.884321+0000","last_clean":"2023-10-09T04:38:50.884321+0000","last_became_active":"2023-10-09T04:34:09.917258+0000","last_became_peered":"2023-10-09T04:34:09.917258+0000","last_unstale":"2023-10-09T04:38:50.884321+0000","last_undegraded":"2023-10-09T04:38:50.884321+0000","last_fullsized":"2023-10-09T04:38:50.884321+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672197+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:38:50.672197+0000","last_peered":"2023-10-09T04:38:50.672197+0000","last_clean":"2023-10-09T04:38:50.672197+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:38:50.672197+0000","last_undegraded":"2023-10-09T04:38:50.672197+0000","last_fullsized":"2023-10-09T04:38:50.672197+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":100,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676952+0000","last_change":"2023-10-09T04:34:09.927188+0000","last_active":"2023-10-09T04:38:50.676952+0000","last_peered":"2023-10-09T04:38:50.676952+0000","last_clean":"2023-10-09T04:38:50.676952+0000","last_became_active":"2023-10-09T04:34:09.926564+0000","last_became_peered":"2023-10-09T04:34:09.926564+0000","last_unstale":"2023-10-09T04:38:50.676952+0000","last_undegraded":"2023-10-09T04:38:50.676952+0000","last_fullsized":"2023-10-09T04:38:50.676952+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"183'6895","reported_seq":6913,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:15.271905+0000","last_change":"2023-10-09T04:37:13.763373+0000","last_active":"2023-10-09T04:39:15.271905+0000","last_peered":"2023-10-09T04:39:15.271905+0000","last_clean":"2023-10-09T04:39:15.271905+0000","last_became_active":"2023-10-09T04:37:13.762984+0000","last_became_peered":"2023-10-09T04:37:13.762984+0000","last_unstale":"2023-10-09T04:39:15.271905+0000","last_undegraded":"2023-10-09T04:39:15.271905+0000","last_fullsized":"2023-10-09T04:39:15.271905+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402587648,"num_objects":6143,"num_object_clones":0,"num_object_copies":18429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6895,"num_write_kb":417216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":727,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.880322+0000","last_change":"2023-10-09T04:22:50.702248+0000","last_active":"2023-10-09T04:38:50.880322+0000","last_peered":"2023-10-09T04:38:50.880322+0000","last_clean":"2023-10-09T04:38:50.880322+0000","last_became_active":"2023-10-09T04:22:50.701344+0000","last_became_peered":"2023-10-09T04:22:50.701344+0000","last_unstale":"2023-10-09T04:38:50.880322+0000","last_undegraded":"2023-10-09T04:38:50.880322+0000","last_fullsized":"2023-10-09T04:38:50.880322+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":175,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672354+0000","last_change":"2023-10-09T04:38:49.685214+0000","last_active":"2023-10-09T04:38:50.672354+0000","last_peered":"2023-10-09T04:38:50.672354+0000","last_clean":"2023-10-09T04:38:50.672354+0000","last_became_active":"2023-10-09T04:38:49.684807+0000","last_became_peered":"2023-10-09T04:38:49.684807+0000","last_unstale":"2023-10-09T04:38:50.672354+0000","last_undegraded":"2023-10-09T04:38:50.672354+0000","last_fullsized":"2023-10-09T04:38:50.672354+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:01.414710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":151,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671537+0000","last_change":"2023-10-09T04:23:18.766574+0000","last_active":"2023-10-09T04:38:49.671537+0000","last_peered":"2023-10-09T04:38:49.671537+0000","last_clean":"2023-10-09T04:38:49.671537+0000","last_became_active":"2023-10-09T04:23:18.766266+0000","last_became_peered":"2023-10-09T04:23:18.766266+0000","last_unstale":"2023-10-09T04:38:49.671537+0000","last_undegraded":"2023-10-09T04:38:49.671537+0000","last_fullsized":"2023-10-09T04:38:49.671537+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:19.321551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"183'6772","reported_seq":6790,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:15.021222+0000","last_change":"2023-10-09T04:37:13.762982+0000","last_active":"2023-10-09T04:39:15.021222+0000","last_peered":"2023-10-09T04:39:15.021222+0000","last_clean":"2023-10-09T04:39:15.021222+0000","last_became_active":"2023-10-09T04:37:13.762606+0000","last_became_peered":"2023-10-09T04:37:13.762606+0000","last_unstale":"2023-10-09T04:39:15.021222+0000","last_undegraded":"2023-10-09T04:39:15.021222+0000","last_fullsized":"2023-10-09T04:39:15.021222+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6772,"log_dups_size":0,"ondisk_log_size":6772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395706368,"num_objects":6038,"num_object_clones":0,"num_object_copies":18114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6772,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":396,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.882107+0000","last_change":"2023-10-09T04:22:23.615272+0000","last_active":"2023-10-09T04:38:50.882107+0000","last_peered":"2023-10-09T04:38:50.882107+0000","last_clean":"2023-10-09T04:38:50.882107+0000","last_became_active":"2023-10-09T04:22:23.614717+0000","last_became_peered":"2023-10-09T04:22:23.614717+0000","last_unstale":"2023-10-09T04:38:50.882107+0000","last_undegraded":"2023-10-09T04:38:50.882107+0000","last_fullsized":"2023-10-09T04:38:50.882107+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711664+0000","last_change":"2023-10-09T04:34:36.642995+0000","last_active":"2023-10-09T04:38:50.711664+0000","last_peered":"2023-10-09T04:38:50.711664+0000","last_clean":"2023-10-09T04:38:50.711664+0000","last_became_active":"2023-10-09T04:34:36.642548+0000","last_became_peered":"2023-10-09T04:34:36.642548+0000","last_unstale":"2023-10-09T04:38:50.711664+0000","last_undegraded":"2023-10-09T04:38:50.711664+0000","last_fullsized":"2023-10-09T04:38:50.711664+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:13.000173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672559+0000","last_change":"2023-10-09T04:34:36.642199+0000","last_active":"2023-10-09T04:38:50.672559+0000","last_peered":"2023-10-09T04:38:50.672559+0000","last_clean":"2023-10-09T04:38:50.672559+0000","last_became_active":"2023-10-09T04:34:36.641868+0000","last_became_peered":"2023-10-09T04:34:36.641868+0000","last_unstale":"2023-10-09T04:38:50.672559+0000","last_undegraded":"2023-10-09T04:38:50.672559+0000","last_fullsized":"2023-10-09T04:38:50.672559+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:45.514132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"182'6641","reported_seq":6658,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:39:11.780453+0000","last_change":"2023-10-09T04:37:13.774838+0000","last_active":"2023-10-09T04:39:11.780453+0000","last_peered":"2023-10-09T04:39:11.780453+0000","last_clean":"2023-10-09T04:39:11.780453+0000","last_became_active":"2023-10-09T04:37:13.774474+0000","last_became_peered":"2023-10-09T04:37:13.774474+0000","last_unstale":"2023-10-09T04:39:11.780453+0000","last_undegraded":"2023-10-09T04:39:11.780453+0000","last_fullsized":"2023-10-09T04:39:11.780453+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6641,"log_dups_size":0,"ondisk_log_size":6641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:10.659935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403243008,"num_objects":6153,"num_object_clones":0,"num_object_copies":18459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6641,"num_write_kb":409408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":97,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.881594+0000","last_change":"2023-10-09T04:28:51.532984+0000","last_active":"2023-10-09T04:38:50.881594+0000","last_peered":"2023-10-09T04:38:50.881594+0000","last_clean":"2023-10-09T04:38:50.881594+0000","last_became_active":"2023-10-09T04:28:51.532480+0000","last_became_peered":"2023-10-09T04:28:51.532480+0000","last_unstale":"2023-10-09T04:38:50.881594+0000","last_undegraded":"2023-10-09T04:38:50.881594+0000","last_fullsized":"2023-10-09T04:38:50.881594+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":424,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.711954+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:38:50.711954+0000","last_peered":"2023-10-09T04:38:50.711954+0000","last_clean":"2023-10-09T04:38:50.711954+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:38:50.711954+0000","last_undegraded":"2023-10-09T04:38:50.711954+0000","last_fullsized":"2023-10-09T04:38:50.711954+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":136,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.880916+0000","last_change":"2023-10-09T04:28:51.526788+0000","last_active":"2023-10-09T04:38:50.880916+0000","last_peered":"2023-10-09T04:38:50.880916+0000","last_clean":"2023-10-09T04:38:50.880916+0000","last_became_active":"2023-10-09T04:28:51.526439+0000","last_became_peered":"2023-10-09T04:28:51.526439+0000","last_unstale":"2023-10-09T04:38:50.880916+0000","last_undegraded":"2023-10-09T04:38:50.880916+0000","last_fullsized":"2023-10-09T04:38:50.880916+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"183'6710","reported_seq":6728,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T04:39:13.801720+0000","last_change":"2023-10-09T04:37:13.767844+0000","last_active":"2023-10-09T04:39:13.801720+0000","last_peered":"2023-10-09T04:39:13.801720+0000","last_clean":"2023-10-09T04:39:13.801720+0000","last_became_active":"2023-10-09T04:37:13.767535+0000","last_became_peered":"2023-10-09T04:37:13.767535+0000","last_unstale":"2023-10-09T04:39:13.801720+0000","last_undegraded":"2023-10-09T04:39:13.801720+0000","last_fullsized":"2023-10-09T04:39:13.801720+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:12.737981+0000","last_clean_scrub_stamp":"2023-10-09T04:37:12.737981+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:31.233224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399376384,"num_objects":6094,"num_object_clones":0,"num_object_copies":18282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6710,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":195,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676994+0000","last_change":"2023-10-09T04:34:09.918839+0000","last_active":"2023-10-09T04:38:50.676994+0000","last_peered":"2023-10-09T04:38:50.676994+0000","last_clean":"2023-10-09T04:38:50.676994+0000","last_became_active":"2023-10-09T04:34:09.918201+0000","last_became_peered":"2023-10-09T04:34:09.918201+0000","last_unstale":"2023-10-09T04:38:50.676994+0000","last_undegraded":"2023-10-09T04:38:50.676994+0000","last_fullsized":"2023-10-09T04:38:50.676994+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":140,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671732+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:38:49.671732+0000","last_peered":"2023-10-09T04:38:49.671732+0000","last_clean":"2023-10-09T04:38:49.671732+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:38:49.671732+0000","last_undegraded":"2023-10-09T04:38:49.671732+0000","last_fullsized":"2023-10-09T04:38:49.671732+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.671983+0000","last_change":"2023-10-09T04:38:49.687481+0000","last_active":"2023-10-09T04:38:50.671983+0000","last_peered":"2023-10-09T04:38:50.671983+0000","last_clean":"2023-10-09T04:38:50.671983+0000","last_became_active":"2023-10-09T04:38:49.687039+0000","last_became_peered":"2023-10-09T04:38:49.687039+0000","last_unstale":"2023-10-09T04:38:50.671983+0000","last_undegraded":"2023-10-09T04:38:50.671983+0000","last_fullsized":"2023-10-09T04:38:50.671983+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":124,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.671569+0000","last_change":"2023-10-09T04:22:23.613370+0000","last_active":"2023-10-09T04:38:49.671569+0000","last_peered":"2023-10-09T04:38:49.671569+0000","last_clean":"2023-10-09T04:38:49.671569+0000","last_became_active":"2023-10-09T04:22:23.611901+0000","last_became_peered":"2023-10-09T04:22:23.611901+0000","last_unstale":"2023-10-09T04:38:49.671569+0000","last_undegraded":"2023-10-09T04:38:49.671569+0000","last_fullsized":"2023-10-09T04:38:49.671569+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:33.431810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.665370+0000","last_change":"2023-10-09T04:34:36.644352+0000","last_active":"2023-10-09T04:38:49.665370+0000","last_peered":"2023-10-09T04:38:49.665370+0000","last_clean":"2023-10-09T04:38:49.665370+0000","last_became_active":"2023-10-09T04:34:36.643870+0000","last_became_peered":"2023-10-09T04:34:36.643870+0000","last_unstale":"2023-10-09T04:38:49.665370+0000","last_undegraded":"2023-10-09T04:38:49.665370+0000","last_fullsized":"2023-10-09T04:38:49.665370+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:32.488036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":108,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672501+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:38:50.672501+0000","last_peered":"2023-10-09T04:38:50.672501+0000","last_clean":"2023-10-09T04:38:50.672501+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:38:50.672501+0000","last_undegraded":"2023-10-09T04:38:50.672501+0000","last_fullsized":"2023-10-09T04:38:50.672501+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":142,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:53.764860+0000","last_change":"2023-10-09T04:38:49.674693+0000","last_active":"2023-10-09T04:38:53.764860+0000","last_peered":"2023-10-09T04:38:53.764860+0000","last_clean":"2023-10-09T04:38:53.764860+0000","last_became_active":"2023-10-09T04:38:49.674306+0000","last_became_peered":"2023-10-09T04:38:49.674306+0000","last_unstale":"2023-10-09T04:38:53.764860+0000","last_undegraded":"2023-10-09T04:38:53.764860+0000","last_fullsized":"2023-10-09T04:38:53.764860+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:33.944564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":101,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672625+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:38:50.672625+0000","last_peered":"2023-10-09T04:38:50.672625+0000","last_clean":"2023-10-09T04:38:50.672625+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:38:50.672625+0000","last_undegraded":"2023-10-09T04:38:50.672625+0000","last_fullsized":"2023-10-09T04:38:50.672625+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.672375+0000","last_change":"2023-10-09T04:38:49.676125+0000","last_active":"2023-10-09T04:38:50.672375+0000","last_peered":"2023-10-09T04:38:50.672375+0000","last_clean":"2023-10-09T04:38:50.672375+0000","last_became_active":"2023-10-09T04:38:49.675696+0000","last_became_peered":"2023-10-09T04:38:49.675696+0000","last_unstale":"2023-10-09T04:38:50.672375+0000","last_undegraded":"2023-10-09T04:38:50.672375+0000","last_fullsized":"2023-10-09T04:38:50.672375+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:32.928438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":70,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.676840+0000","last_change":"2023-10-09T04:34:09.901145+0000","last_active":"2023-10-09T04:38:50.676840+0000","last_peered":"2023-10-09T04:38:50.676840+0000","last_clean":"2023-10-09T04:38:50.676840+0000","last_became_active":"2023-10-09T04:34:09.900787+0000","last_became_peered":"2023-10-09T04:34:09.900787+0000","last_unstale":"2023-10-09T04:38:50.676840+0000","last_undegraded":"2023-10-09T04:38:50.676840+0000","last_fullsized":"2023-10-09T04:38:50.676840+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":163,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.665152+0000","last_change":"2023-10-09T04:28:51.527544+0000","last_active":"2023-10-09T04:38:49.665152+0000","last_peered":"2023-10-09T04:38:49.665152+0000","last_clean":"2023-10-09T04:38:49.665152+0000","last_became_active":"2023-10-09T04:28:51.527179+0000","last_became_peered":"2023-10-09T04:28:51.527179+0000","last_unstale":"2023-10-09T04:38:49.665152+0000","last_undegraded":"2023-10-09T04:38:49.665152+0000","last_fullsized":"2023-10-09T04:38:49.665152+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:12.587447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":156,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.673116+0000","last_change":"2023-10-09T04:34:36.645881+0000","last_active":"2023-10-09T04:38:50.673116+0000","last_peered":"2023-10-09T04:38:50.673116+0000","last_clean":"2023-10-09T04:38:50.673116+0000","last_became_active":"2023-10-09T04:34:36.644899+0000","last_became_peered":"2023-10-09T04:34:36.644899+0000","last_unstale":"2023-10-09T04:38:50.673116+0000","last_undegraded":"2023-10-09T04:38:50.673116+0000","last_fullsized":"2023-10-09T04:38:50.673116+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:11.020741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":162,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.713982+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:38:50.713982+0000","last_peered":"2023-10-09T04:38:50.713982+0000","last_clean":"2023-10-09T04:38:50.713982+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:38:50.713982+0000","last_undegraded":"2023-10-09T04:38:50.713982+0000","last_fullsized":"2023-10-09T04:38:50.713982+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6434586648,"num_objects":98185,"num_object_clones":0,"num_object_copies":294555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11551,"num_objects_misplaced":5448,"num_objects_unfound":0,"num_objects_dirty":98185,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107131,"num_write_kb":6570049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":321,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19268448256,"data_stored":19268436040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107131,"ondisk_log_size":107131,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41128,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430578,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2304944,"kb_used_data":1995432,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91440208,"statfs":{"total":95995035648,"available":93634772992,"internally_reserved":0,"allocated":2043322368,"data_stored":2042777110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:39:00 2023","interfaces":[{"interface":"back","average":{"1min":6.3390000000000004,"5min":2.3069999999999999,"15min":2.2629999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":104.16200000000001,"5min":104.16200000000001,"15min":250.65700000000001},"last":8.1029999999999998},{"interface":"front","average":{"1min":5.9619999999999997,"5min":2.5569999999999999,"15min":2.661},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.23400000000000001},"max":{"1min":104.129,"5min":104.129,"15min":251.214},"last":3.3260000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:38:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.3120000000000001,"15min":1.1080000000000001},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.7649999999999997,"5min":5.9290000000000003,"15min":7.282},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.589,"15min":1.415},"min":{"1min":0.34399999999999997,"5min":0.33900000000000002,"15min":0.28299999999999997},"max":{"1min":4.8289999999999997,"5min":11.555999999999999,"15min":11.988},"last":3.8210000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:39:00 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.6439999999999999,"15min":1.752},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.1970000000000001,"5min":8.5120000000000005,"15min":106.456},"last":3.1200000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.5029999999999999,"15min":1.722},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.6900000000000004,"5min":7.4379999999999997,"15min":106.708},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:38:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":1.782,"15min":1.3540000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.318},"max":{"1min":4.6130000000000004,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.395,"5min":1.5429999999999999,"15min":1.2809999999999999},"min":{"1min":0.38500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.6740000000000004,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":0.65000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:38:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.3720000000000001,"15min":1.28},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":4.7939999999999996,"5min":11.683999999999999,"15min":11.683999999999999},"last":0.437},{"interface":"front","average":{"1min":1.375,"5min":1.4550000000000001,"15min":1.1879999999999999},"min":{"1min":0.36899999999999999,"5min":0.34699999999999998,"15min":0.317},"max":{"1min":4.2400000000000002,"5min":10.696999999999999,"15min":10.696999999999999},"last":0.48099999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.6160000000000001,"15min":1.8899999999999999},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.2610000000000001,"5min":8.0790000000000006,"15min":8.0790000000000006},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.8120000000000001,"15min":2.2080000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.798,"5min":10.786,"15min":10.786},"last":1.2629999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:39:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.5549999999999999,"15min":1.6699999999999999},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.5640000000000001,"5min":11.955,"15min":11.955},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.167,"5min":1.21,"15min":1.1899999999999999},"min":{"1min":0.38300000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.5579999999999998,"5min":4.351,"15min":4.351},"last":0.74399999999999999}]}]},{"osd":6,"up_from":146,"seq":627065225278,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2340900,"kb_used_data":2006748,"kb_used_omap":5,"kb_used_meta":334138,"kb_avail":91404252,"statfs":{"total":95995035648,"available":93597954048,"internally_reserved":0,"allocated":2054909952,"data_stored":2054369010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":342157920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":3.0499999999999998,"15min":2.5449999999999999},"min":{"1min":0.34699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":4.6349999999999998,"5min":125.666,"15min":125.666},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.222,"5min":2.976,"15min":2.5110000000000001},"min":{"1min":0.40600000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.4079999999999999,"5min":124.991,"15min":124.991},"last":3.8380000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":2.7290000000000001,"15min":4.0190000000000001},"min":{"1min":0.39800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.9510000000000001,"5min":68.155000000000001,"15min":68.155000000000001},"last":3.1299999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":3.2080000000000002,"15min":4.7039999999999997},"min":{"1min":0.52900000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.6449999999999996,"5min":67.308999999999997,"15min":67.308999999999997},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":3.0880000000000001,"15min":4.718},"min":{"1min":0.439,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.2589999999999999,"5min":68.316000000000003,"15min":68.316000000000003},"last":3.9900000000000002},{"interface":"front","average":{"1min":1.51,"5min":3.2200000000000002,"15min":4.5979999999999999},"min":{"1min":0.439,"5min":0.313,"15min":0.313},"max":{"1min":4.1619999999999999,"5min":68.210999999999999,"15min":68.210999999999999},"last":1.1259999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":2.024,"15min":2.0720000000000001},"min":{"1min":0.44500000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.4930000000000003,"5min":11.673,"15min":11.673},"last":2.0680000000000001},{"interface":"front","average":{"1min":1.863,"5min":2.2149999999999999,"15min":2.4089999999999998},"min":{"1min":0.49399999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":7.6050000000000004,"5min":15.824,"15min":15.824},"last":1.998}]},{"osd":4,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.3460000000000001,"15min":1.5760000000000001},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.0720000000000001,"5min":8.0500000000000007,"15min":8.0500000000000007},"last":3.7959999999999998},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.4710000000000001,"15min":1.4770000000000001},"min":{"1min":0.48599999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.0379999999999998,"5min":10.324,"15min":10.324},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.6759999999999999,"15min":2.3500000000000001},"min":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.798,"5min":10.193,"15min":10.193},"last":3.915},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.839,"15min":2.2709999999999999},"min":{"1min":0.44500000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":3.0800000000000001,"5min":11.727,"15min":11.727},"last":0.60399999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.758,"15min":2.0139999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.8390000000000004,"5min":13.191000000000001,"15min":13.191000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.526,"15min":1.7050000000000001},"min":{"1min":0.56699999999999995,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.133,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":3.8799999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958727,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2681900,"kb_used_data":2378980,"kb_used_omap":6,"kb_used_meta":302777,"kb_avail":91063252,"statfs":{"total":95995035648,"available":93248770048,"internally_reserved":0,"allocated":2436075520,"data_stored":2435302165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":310043660},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:20 2023","interfaces":[{"interface":"back","average":{"1min":7.9320000000000004,"5min":2.7210000000000001,"15min":2.4199999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":48.954000000000001,"5min":48.954000000000001,"15min":158.435},"last":0.34899999999999998},{"interface":"front","average":{"1min":8.4540000000000006,"5min":2.8149999999999999,"15min":2.5659999999999998},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":48.606000000000002,"5min":48.606000000000002,"15min":159.38999999999999},"last":0.24399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:38:55 2023","interfaces":[{"interface":"back","average":{"1min":9.9299999999999997,"5min":3.0950000000000002,"15min":2.4980000000000002},"min":{"1min":0.35099999999999998,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":90.275999999999996,"5min":90.275999999999996,"15min":157.70400000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":10.630000000000001,"5min":3.218,"15min":2.5859999999999999},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":90.409000000000006,"5min":90.409000000000006,"15min":157.64099999999999},"last":1.8979999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":1.9590000000000001,"15min":1.5},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.5619999999999994,"5min":14.369,"15min":14.369},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.3769999999999998,"5min":2.024,"15min":1.4570000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.4030000000000005,"5min":12.932,"15min":12.932},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":1.212,"15min":1.272},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.0569999999999999,"5min":12.327,"15min":12.327},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.2390000000000001,"15min":1.327},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.5950000000000002,"5min":12.209,"15min":12.209},"last":0.65400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:38:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":1.5900000000000001,"15min":1.7989999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0599999999999996,"5min":6.9900000000000002,"15min":6.9900000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.5229999999999999,"15min":1.8560000000000001},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.2389999999999999,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6930000000000001,"15min":1.994},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.2329999999999997,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":1.865},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.7410000000000001,"15min":2.0329999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.258,"5min":8.9879999999999995,"15min":8.9879999999999995},"last":1.911}]},{"osd":7,"last update":"Mon Oct 9 04:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":2.6640000000000001,"15min":1.7529999999999999},"min":{"1min":0.44400000000000001,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":2.923,"5min":144.613,"15min":144.613},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.4670000000000001,"5min":2.556,"15min":1.764},"min":{"1min":0.42999999999999999,"5min":0.41699999999999998,"15min":0.39100000000000001},"max":{"1min":4.6210000000000004,"5min":145.53700000000001,"15min":145.53700000000001},"last":0.625}]}]},{"osd":0,"up_from":88,"seq":377957122252,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3537968,"kb_used_data":3183336,"kb_used_omap":9,"kb_used_meta":354550,"kb_avail":90207184,"statfs":{"total":95995035648,"available":92372156416,"internally_reserved":0,"allocated":3259736064,"data_stored":3258954098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":363059697},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:38:27 2023","interfaces":[{"interface":"back","average":{"1min":9.9390000000000001,"5min":3.4359999999999999,"15min":2.0190000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":223.98500000000001,"5min":223.98500000000001,"15min":223.98500000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":10.259,"5min":3.1150000000000002,"15min":1.9810000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":223.13,"5min":223.13,"15min":223.13},"last":5.5590000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":2.3410000000000002,"15min":1.748},"min":{"1min":0.26400000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.194000000000001,"5min":61.634999999999998,"15min":61.634999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.972,"15min":1.486},"min":{"1min":0.32500000000000001,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":10.164,"5min":64.677000000000007,"15min":64.677000000000007},"last":8.3059999999999992}]},{"osd":3,"last update":"Mon Oct 9 04:38:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":2.1859999999999999,"15min":1.3520000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.20300000000000001},"max":{"1min":13.119999999999999,"5min":64.622,"15min":64.622},"last":0.73899999999999999},{"interface":"front","average":{"1min":2.1160000000000001,"5min":2.3050000000000002,"15min":1.4790000000000001},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":9.0120000000000005,"5min":64.540999999999997,"15min":64.540999999999997},"last":4.3730000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.974,"15min":1.55},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4340000000000002,"5min":61.549999999999997,"15min":61.549999999999997},"last":4.46},{"interface":"front","average":{"1min":1.631,"5min":1.8480000000000001,"15min":1.351},"min":{"1min":0.317,"5min":0.317,"15min":0.308},"max":{"1min":10.218999999999999,"5min":61.902999999999999,"15min":61.902999999999999},"last":0.71399999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.5550000000000002,"15min":3.5249999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.202,"5min":61.972999999999999,"15min":61.972999999999999},"last":4.3390000000000004},{"interface":"front","average":{"1min":1.5660000000000001,"5min":2.786,"15min":4.0819999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.2439999999999998,"5min":62.417000000000002,"15min":62.417000000000002},"last":0.48299999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":2.7440000000000002,"15min":3.6930000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.2990000000000004,"5min":62.462000000000003,"15min":62.462000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.9369999999999998,"15min":3.9750000000000001},"min":{"1min":0.44400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6760000000000002,"5min":61.808,"15min":61.808},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":2.2869999999999999,"15min":1.7589999999999999},"min":{"1min":0.46999999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.6550000000000002,"5min":62.006,"15min":62.006},"last":4.3029999999999999},{"interface":"front","average":{"1min":1.667,"5min":2.2829999999999999,"15min":1.768},"min":{"1min":0.39600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.5540000000000003,"5min":61.606999999999999,"15min":61.606999999999999},"last":4.524}]}]},{"osd":2,"up_from":98,"seq":420906795201,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2761116,"kb_used_data":2401300,"kb_used_omap":11,"kb_used_meta":359796,"kb_avail":90984036,"statfs":{"total":95995035648,"available":93167652864,"internally_reserved":0,"allocated":2458931200,"data_stored":2458165897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":368431146},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.677,"15min":1.248},"min":{"1min":0.28399999999999997,"5min":0.22900000000000001,"15min":0.191},"max":{"1min":10.959,"5min":18.155999999999999,"15min":18.155999999999999},"last":1.728},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.7310000000000001,"15min":1.3300000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":19.219000000000001,"5min":19.219000000000001,"15min":19.219000000000001},"last":0.44800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":10.847,"5min":3.2589999999999999,"15min":1.796},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":189.14099999999999,"5min":189.14099999999999,"15min":189.14099999999999},"last":1.6850000000000001},{"interface":"front","average":{"1min":10.776999999999999,"5min":3.2519999999999998,"15min":1.923},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":190.023,"5min":190.023,"15min":190.023},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:37 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.7470000000000001,"15min":1.3080000000000001},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.27800000000000002},"max":{"1min":6.0629999999999997,"5min":14.515000000000001,"15min":14.515000000000001},"last":1.8149999999999999},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.6599999999999999,"15min":1.373},"min":{"1min":0.26600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.7519999999999998,"5min":14.66,"15min":14.66},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.3400000000000001,"15min":1.3480000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.29599999999999999},"max":{"1min":2.5379999999999998,"5min":13.689,"15min":13.689},"last":0.63600000000000001},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.798,"15min":1.631},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.33500000000000002},"max":{"1min":7.032,"5min":13.759,"15min":13.759},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:38:43 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":2.5070000000000001,"15min":3.0329999999999999},"min":{"1min":0.35999999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.0650000000000004,"5min":14.564,"15min":14.564},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.452,"5min":1.669,"15min":1.972},"min":{"1min":0.33800000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.9969999999999999,"5min":14.305999999999999,"15min":14.305999999999999},"last":1.8420000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.5580000000000001,"15min":1.601},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":2.8260000000000001,"5min":9.5719999999999992,"15min":9.5719999999999992},"last":1.7609999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.5409999999999999,"15min":1.413},"min":{"1min":0.46700000000000003,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":5.069,"5min":14.683999999999999,"15min":14.683999999999999},"last":2.7919999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.6910000000000001,"15min":1.4259999999999999},"min":{"1min":0.502,"5min":0.45200000000000001,"15min":0.35099999999999998},"max":{"1min":6.0309999999999997,"5min":14.372999999999999,"15min":14.372999999999999},"last":1.788},{"interface":"front","average":{"1min":1.278,"5min":1.5660000000000001,"15min":1.4370000000000001},"min":{"1min":0.41799999999999998,"5min":0.38400000000000001,"15min":0.29899999999999999},"max":{"1min":3.1899999999999999,"5min":14.6,"15min":14.6},"last":0.53400000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631673,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3858420,"kb_used_data":3513452,"kb_used_omap":5,"kb_used_meta":344954,"kb_avail":89886732,"statfs":{"total":95995035648,"available":92044013568,"internally_reserved":0,"allocated":3597774848,"data_stored":3596994433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":353233070},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2290000000000001,"5min":1.431,"15min":1.5049999999999999},"min":{"1min":0.33700000000000002,"5min":0.154,"15min":0.154},"max":{"1min":6.875,"5min":9.6379999999999999,"15min":61.966999999999999},"last":6.2290000000000001},{"interface":"front","average":{"1min":1.772,"5min":1.4570000000000001,"15min":1.5349999999999999},"min":{"1min":0.252,"5min":0.221,"15min":0.221},"max":{"1min":6.5220000000000002,"5min":12.638999999999999,"15min":62.052999999999997},"last":6.6379999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.1060000000000001,"15min":1.248},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":2.5230000000000001,"5min":11.920999999999999,"15min":61.395000000000003},"last":3.8980000000000001},{"interface":"front","average":{"1min":1.655,"5min":1.179,"15min":1.27},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":7.7839999999999998,"5min":7.7839999999999998,"15min":61.179000000000002},"last":4.0090000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":1.7509999999999999,"15min":2.161},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":11.297000000000001,"5min":12.087999999999999,"15min":181.959},"last":4.9379999999999997},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.577,"15min":2.105},"min":{"1min":0.42299999999999999,"5min":0.24199999999999999,"15min":0.23799999999999999},"max":{"1min":10.663,"5min":10.663,"15min":181.91200000000001},"last":4.8479999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.266,"15min":1.1100000000000001},"min":{"1min":0.45000000000000001,"5min":0.33700000000000002,"15min":0.314},"max":{"1min":4.3879999999999999,"5min":7.3520000000000003,"15min":7.3520000000000003},"last":4.101},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.319,"15min":1.1870000000000001},"min":{"1min":0.41299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4390000000000001,"5min":7.9260000000000002,"15min":7.9260000000000002},"last":5.5860000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":1.639,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.7750000000000004,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":5.6230000000000002},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.968,"15min":2.4420000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8220000000000001,"5min":7.976,"15min":7.976},"last":5.5499999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:38:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":2.0659999999999998,"15min":2.355},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0739999999999998,"5min":11.842000000000001,"15min":11.842000000000001},"last":3.988},{"interface":"front","average":{"1min":1.53,"5min":1.6739999999999999,"15min":1.7749999999999999},"min":{"1min":0.44900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.3559999999999999,"5min":8.5020000000000007,"15min":8.5020000000000007},"last":4.1379999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":4.3239999999999998,"15min":2.2069999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.6059999999999999,"5min":198.012,"15min":198.012},"last":4.9109999999999996},{"interface":"front","average":{"1min":1.802,"5min":4.101,"15min":2.1539999999999999},"min":{"1min":0.51100000000000001,"5min":0.47399999999999998,"15min":0.34599999999999997},"max":{"1min":7.9050000000000002,"5min":198.05199999999999,"15min":198.05199999999999},"last":4.8769999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019275,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2193320,"kb_used_data":1892000,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91551832,"statfs":{"total":95995035648,"available":93749075968,"internally_reserved":0,"allocated":1937408000,"data_stored":1936870320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.343,"15min":1.4530000000000001},"min":{"1min":0.29399999999999998,"5min":0.26100000000000001,"15min":0.24099999999999999},"max":{"1min":22.841000000000001,"5min":22.841000000000001,"15min":48.450000000000003},"last":0.32900000000000001},{"interface":"front","average":{"1min":3.101,"5min":1.9139999999999999,"15min":1.7270000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":22.966000000000001,"5min":22.966000000000001,"15min":49.283000000000001},"last":2.1269999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.196,"15min":1.514},"min":{"1min":0.308,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":4.9569999999999999,"5min":10.459,"15min":79.361999999999995},"last":5.2350000000000003},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.2490000000000001,"15min":1.639},"min":{"1min":0.48199999999999998,"5min":0.316,"15min":0.29199999999999998},"max":{"1min":5.3929999999999998,"5min":10.487,"15min":79.292000000000002},"last":0.66200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.6000000000000001,"15min":1.7889999999999999},"min":{"1min":0.32500000000000001,"5min":0.31900000000000001,"15min":0.31},"max":{"1min":4.8300000000000001,"5min":13.285,"15min":77.537000000000006},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.2509999999999999,"15min":1.821},"min":{"1min":0.40200000000000002,"5min":0.36299999999999999,"15min":0.311},"max":{"1min":6.0979999999999999,"5min":11.500999999999999,"15min":77.188000000000002},"last":0.57699999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.4870000000000001,"15min":1.2549999999999999},"min":{"1min":0.46300000000000002,"5min":0.38900000000000001,"15min":0.317},"max":{"1min":7.048,"5min":10.417,"15min":10.417},"last":4.4969999999999999},{"interface":"front","average":{"1min":1.3859999999999999,"5min":1.472,"15min":1.339},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.992,"5min":8.7899999999999991,"15min":11.112},"last":4.4820000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.083,"15min":1.083},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.4990000000000001,"5min":10.412000000000001,"15min":10.412000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.995,"5min":1.4650000000000001,"15min":1.661},"min":{"1min":0.38100000000000001,"5min":0.378,"15min":0.378},"max":{"1min":3.9700000000000002,"5min":6.4470000000000001,"15min":6.4470000000000001},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.05,"15min":0.92000000000000004},"min":{"1min":0.42799999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.8340000000000001,"5min":3.8340000000000001,"15min":3.8340000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.2390000000000001,"15min":1.2529999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.635,"5min":10.257999999999999,"15min":10.257999999999999},"last":0.51600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.0629999999999999,"15min":1.216},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.526,"5min":8.734,"15min":11.725},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.413,"15min":1.2490000000000001},"min":{"1min":0.42899999999999999,"5min":0.42399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6150000000000002,"5min":13.349,"15min":13.349},"last":0.53900000000000003}]}]},{"osd":5,"up_from":150,"seq":644245094457,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1903064,"kb_used_data":1553360,"kb_used_omap":6,"kb_used_meta":349689,"kb_avail":91842088,"statfs":{"total":95995035648,"available":94046298112,"internally_reserved":0,"allocated":1590640640,"data_stored":1590114792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":358081811},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":1.6759999999999999,"15min":1.784},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.79,"5min":1.806,"15min":2.016},"min":{"1min":0.36499999999999999,"5min":0.315,"15min":0.315},"max":{"1min":6.9690000000000003,"5min":8.2319999999999993,"15min":8.2319999999999993},"last":0.61799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.768,"15min":2.1240000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.976,"5min":7.2750000000000004,"15min":7.2750000000000004},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.667,"15min":1.8009999999999999},"min":{"1min":0.32900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.9429999999999996,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":3.488}]},{"osd":2,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.0219999999999998,"15min":2.359},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":9.1449999999999996,"5min":12.69,"15min":12.69},"last":4.8090000000000002},{"interface":"front","average":{"1min":1.522,"5min":1.881,"15min":2.1480000000000001},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.9290000000000003,"5min":10.657999999999999,"15min":10.657999999999999},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.611,"15min":1.8520000000000001},"min":{"1min":0.41599999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":6.1559999999999997,"5min":7.3250000000000002,"15min":7.3250000000000002},"last":2.29},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.157,"15min":2.629},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.9130000000000003,"5min":8.2859999999999996,"15min":8.2859999999999996},"last":1.472}]},{"osd":4,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.6140000000000001,"15min":1.9299999999999999},"min":{"1min":0.41199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.8550000000000004,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.294,"15min":1.526},"min":{"1min":0.36499999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.6479999999999997,"5min":6.8300000000000001,"15min":6.8300000000000001},"last":0.375}]},{"osd":6,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.5069999999999999,"15min":1.7849999999999999},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.5060000000000002,"5min":11.087,"15min":11.087},"last":2.6579999999999999},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.641,"15min":2.0019999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.181,"5min":9.3659999999999997,"15min":9.3659999999999997},"last":0.50800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.2149999999999999,"15min":2.2959999999999998},"min":{"1min":0.441,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":5.2430000000000003,"5min":39.771000000000001,"15min":39.771000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.044,"5min":2.3119999999999998,"15min":2.9129999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":3.5289999999999999,"5min":39.905000000000001,"15min":39.905000000000001},"last":0.52100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3240038400,"data_stored":3240034328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2430210048,"data_stored":2430205976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2441019392,"data_stored":2441019392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3570728960,"data_stored":3570728960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1931149312,"data_stored":1931149312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1577910272,"data_stored":1577910272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2045251584,"data_stored":2045247512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2032140288,"data_stored":2032140288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:39:20.222 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:39:20.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-10-09T04:39:20.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:20.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:20.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:20.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:20.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:20.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:20.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:20.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:20.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:20.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:20.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:20.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:20.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:21.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:21.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:21.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:21.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:21.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:21.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:21.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:21.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:21.840 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:21.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:21 smithi082 bash[77063]: cluster 2023-10-09T04:39:20.472780+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 11639/284448 objects degraded (4.092%); 5292/284448 objects misplaced (1.860%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:21.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:21 smithi082 bash[79733]: cluster 2023-10-09T04:39:20.472780+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 11639/284448 objects degraded (4.092%); 5292/284448 objects misplaced (1.860%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:21.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:21.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:22.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:22.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:21 smithi153 bash[59565]: cluster 2023-10-09T04:39:20.472780+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 11639/284448 objects degraded (4.092%); 5292/284448 objects misplaced (1.860%); 966 KiB/s, 15 objects/s recovering 2023-10-09T04:39:22.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:22.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:22.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:22.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:22.422 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:22.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:22.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:22.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:22.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:22.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:22.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:22.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:22.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:22.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:22.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:22.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:23.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:23.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:23.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:23.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:23.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:23.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:23.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:23.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:23.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:23.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:23.877 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114442 7.3 7.6a704813 (undecoded) ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.889546+0000", 2023-10-09T04:39:23.894 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0020371539999999998, 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002063488, 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114442 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.895 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889546+0000", 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889546+0000", 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889545+0000", 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:23.896 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889551+0000", 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.891e-06 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889562+0000", 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1586000000000001e-05 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.897 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889570+0000", 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1910000000000003e-06 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114443 7.3 7.55873343 (undecoded) ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.898 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.889894+0000", 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.001688953, 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001777722, 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114443 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.899 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889894+0000", 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889894+0000", 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.900 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889894+0000", 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889899+0000", 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8210000000000001e-06 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889909+0000", 2023-10-09T04:39:23.901 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0786e-05 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.889916+0000", 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.973e-06 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.902 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114428 7.8 7:1a44e43c:::benchmark_data_smithi082_155217_object11768:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.865825+0000", 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.025758548999999999, 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.025904488999999999, 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114428 2023-10-09T04:39:23.903 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865825+0000", 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865825+0000", 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.904 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865824+0000", 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865827+0000", 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8590000000000001e-06 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.905 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865834+0000", 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2620000000000001e-06 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.865840+0000", 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1959999999999999e-06 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.890178+0000", 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.024338529000000001 2023-10-09T04:39:23.906 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.890306+0000", 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012814400000000001 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.890413+0000", 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010716499999999999 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.907 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114431 7.8 7.6da0caf8 (undecoded) ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.870828+0000", 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.020755644, 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.020970874, 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.908 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114431 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870828+0000", 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870828+0000", 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.909 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870827+0000", 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870833+0000", 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6460000000000004e-06 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.910 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870842+0000", 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7830000000000004e-06 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.870850+0000", 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5399999999999998e-06 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.911 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114434 7.3 7:c0090fe2:::benchmark_data_smithi082_155217_object11774:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.879186+0000", 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.012397803000000001, 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012745497999999999, 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:39:23.912 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114434 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879186+0000", 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.913 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879186+0000", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879184+0000", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879192+0000", 2023-10-09T04:39:23.914 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7479999999999999e-06 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879422+0000", 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000229177 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879433+0000", 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1344e-05 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.915 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879494+0000", 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0794000000000001e-05 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879638+0000", 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000144704 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:39:23.916 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.879842+0000", 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020320200000000001 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.883903+0000", 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040613389999999997 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.885803+0000", 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019002330000000001 2023-10-09T04:39:23.917 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:114439 7.3 7:cd7c4fb9:::benchmark_data_smithi082_155217_object11779:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:39:23.884362+0000", 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0072214499999999999, 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0076692009999999996, 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:39:23.918 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 114439 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884362+0000", 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:39:23.919 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884362+0000", 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884360+0000", 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884368+0000", 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7090000000000002e-06 2023-10-09T04:39:23.920 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884385+0000", 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6906e-05 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.884393+0000", 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4549999999999995e-06 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:39:23.921 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.885833+0000", 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00143958 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.885946+0000", 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000112934 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:39:23.886056+0000", 2023-10-09T04:39:23.922 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000110717 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 6 2023-10-09T04:39:23.923 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:39:23.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:23.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:23.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:23.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:23 smithi153 bash[59565]: cluster 2023-10-09T04:39:22.473449+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11749/279612 objects degraded (4.202%); 5101/279612 objects misplaced (1.824%); 953 KiB/s, 14 objects/s recovering 2023-10-09T04:39:24.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:24.033 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:23 smithi082 bash[77063]: cluster 2023-10-09T04:39:22.473449+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11749/279612 objects degraded (4.202%); 5101/279612 objects misplaced (1.824%); 953 KiB/s, 14 objects/s recovering 2023-10-09T04:39:24.033 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:23 smithi082 bash[79733]: cluster 2023-10-09T04:39:22.473449+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11749/279612 objects degraded (4.202%); 5101/279612 objects misplaced (1.824%); 953 KiB/s, 14 objects/s recovering 2023-10-09T04:39:24.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:24.165 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:24.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:24.316 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:24.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:24.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:24.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:24.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:24.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:24.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:24.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:24.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:24.744 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:24.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:24.860 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:24 smithi153 bash[59565]: cluster 2023-10-09T04:39:24.679108+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 11749/279612 objects degraded (4.202%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:24.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:24 smithi082 bash[77063]: cluster 2023-10-09T04:39:24.679108+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 11749/279612 objects degraded (4.202%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:24 smithi082 bash[79733]: cluster 2023-10-09T04:39:24.679108+0000 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 11749/279612 objects degraded (4.202%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:25.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:25.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:25.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:25.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:25.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:39:25.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:25.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:25.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:25.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:25.669 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.694 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:25.763 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:25.763 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:25.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.829 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:25.830 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:25.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:25.974 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:25.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:26.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:26.089 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:25 smithi153 bash[59565]: cluster 2023-10-09T04:39:24.474244+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11749/276000 objects degraded (4.257%); 5101/276000 objects misplaced (1.848%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:26.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:26.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:26.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:25 smithi082 bash[77063]: cluster 2023-10-09T04:39:24.474244+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11749/276000 objects degraded (4.257%); 5101/276000 objects misplaced (1.848%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:26.228 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:25 smithi082 bash[79733]: cluster 2023-10-09T04:39:24.474244+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11749/276000 objects degraded (4.257%); 5101/276000 objects misplaced (1.848%); 810 KiB/s, 12 objects/s recovering 2023-10-09T04:39:26.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:26.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:26.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:26.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:26.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:26.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:26.626 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:26.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:26.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:26.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:26.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:26.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:26.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:27.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:27.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:27.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:27.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:27.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:27.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:27.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:27.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:27.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:27.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:27.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:27.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:27.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:27.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:27 smithi082 bash[77063]: cluster 2023-10-09T04:39:26.476849+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11749/274704 objects degraded (4.277%); 5101/274704 objects misplaced (1.857%); 794 KiB/s, 12 objects/s recovering 2023-10-09T04:39:27.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:27 smithi082 bash[79733]: cluster 2023-10-09T04:39:26.476849+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11749/274704 objects degraded (4.277%); 5101/274704 objects misplaced (1.857%); 794 KiB/s, 12 objects/s recovering 2023-10-09T04:39:27.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:28.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:27 smithi153 bash[59565]: cluster 2023-10-09T04:39:26.476849+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11749/274704 objects degraded (4.277%); 5101/274704 objects misplaced (1.857%); 794 KiB/s, 12 objects/s recovering 2023-10-09T04:39:28.126 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:28.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:28.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:28.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:28.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:28.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:28.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:28.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:28.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:28.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:28.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:28.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:28.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:28.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:28.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:29.022 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:29.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:29.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:29.183 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:29.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:29.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:29.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:29.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:29.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:29.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:29.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:29.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:29.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:29.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:29.831 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:29] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[77063]: cluster 2023-10-09T04:39:28.477523+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11886/267447 objects degraded (4.444%); 4889/267447 objects misplaced (1.828%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[77063]: audit 2023-10-09T04:39:29.277782+0000 mon.a (mon.0) 1100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[77063]: cluster 2023-10-09T04:39:29.680184+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 11886/267447 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[79733]: cluster 2023-10-09T04:39:28.477523+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11886/267447 objects degraded (4.444%); 4889/267447 objects misplaced (1.828%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[79733]: audit 2023-10-09T04:39:29.277782+0000 mon.a (mon.0) 1100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:29.832 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:29 smithi082 bash[79733]: cluster 2023-10-09T04:39:29.680184+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 11886/267447 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:29.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:29.900 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:29.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:29.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:29.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:30.088 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:29 smithi153 bash[59565]: cluster 2023-10-09T04:39:28.477523+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11886/267447 objects degraded (4.444%); 4889/267447 objects misplaced (1.828%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:39:30.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:29 smithi153 bash[59565]: audit 2023-10-09T04:39:29.277782+0000 mon.a (mon.0) 1100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:30.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:29 smithi153 bash[59565]: cluster 2023-10-09T04:39:29.680184+0000 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 11886/267447 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:30.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:30.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:30.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:30.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:30.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:30.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:30.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:30.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:30.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:30.817 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:30.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:30.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:30.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:31.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:31.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:31.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:31.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:31.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:31.441 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:31.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:31.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:31.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:31.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:32.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:32.016 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:31 smithi082 bash[77063]: cluster 2023-10-09T04:39:30.478182+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11886/265161 objects degraded (4.483%); 4889/265161 objects misplaced (1.844%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:39:32.017 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:31 smithi082 bash[79733]: cluster 2023-10-09T04:39:30.478182+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11886/265161 objects degraded (4.483%); 4889/265161 objects misplaced (1.844%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:39:32.049 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:32.153 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:32.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:31 smithi153 bash[59565]: cluster 2023-10-09T04:39:30.478182+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11886/265161 objects degraded (4.483%); 4889/265161 objects misplaced (1.844%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:39:32.176 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:32.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:32.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:32.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:32.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:32.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:32.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:32.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:32.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:32.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:32.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:32.941 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:32.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:32.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:33.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:33.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:33.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:33.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:33.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:33.369 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:33.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:33.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:33.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:33.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:33.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:33.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:33.885 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:33 smithi153 bash[59565]: cluster 2023-10-09T04:39:32.478762+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 12002/260277 objects degraded (4.611%); 4687/260277 objects misplaced (1.801%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:33.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:33.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:34.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:34.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:34.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:34.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:33 smithi082 bash[77063]: cluster 2023-10-09T04:39:32.478762+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 12002/260277 objects degraded (4.611%); 4687/260277 objects misplaced (1.801%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:34.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:33 smithi082 bash[79733]: cluster 2023-10-09T04:39:32.478762+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 12002/260277 objects degraded (4.611%); 4687/260277 objects misplaced (1.801%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:34.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:34.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:34.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:34.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:34.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:34.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:34.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:34.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:34.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:34.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:34.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:34.830 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:34 smithi082 bash[79733]: cluster 2023-10-09T04:39:34.681421+0000 mon.a (mon.0) 1102 : cluster 3 Health check update: Degraded data redundancy: 12002/260277 objects degraded (4.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:34.830 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:34 smithi082 bash[77063]: cluster 2023-10-09T04:39:34.681421+0000 mon.a (mon.0) 1102 : cluster 3 Health check update: Degraded data redundancy: 12002/260277 objects degraded (4.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:34.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:34.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:34.960 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:34.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:34.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:35.094 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:35.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:34 smithi153 bash[59565]: cluster 2023-10-09T04:39:34.681421+0000 mon.a (mon.0) 1102 : cluster 3 Health check update: Degraded data redundancy: 12002/260277 objects degraded (4.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:35.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:35.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.251 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:35.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:35.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:35.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:35.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:35.618 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:35.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:35.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:35.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:35.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:35.889 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:35 smithi082 bash[77063]: cluster 2023-10-09T04:39:34.479338+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12002/256545 objects degraded (4.678%); 4687/256545 objects misplaced (1.827%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:35.889 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:35 smithi082 bash[79733]: cluster 2023-10-09T04:39:34.479338+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12002/256545 objects degraded (4.678%); 4687/256545 objects misplaced (1.827%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:35.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:35.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:36.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:36.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:36.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:36.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:35 smithi153 bash[59565]: cluster 2023-10-09T04:39:34.479338+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12002/256545 objects degraded (4.678%); 4687/256545 objects misplaced (1.827%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:36.195 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:36.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:36.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:36.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:36.478 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:36.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:36.617 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:36.724 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:36.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:36.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:36.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:36.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:37.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:37.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:37.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:37.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:37.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:37.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:37.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:37.541 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:37.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:37.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:37.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:37.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:37.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.939 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:37 smithi082 bash[77063]: cluster 2023-10-09T04:39:36.480091+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 12002/255300 objects degraded (4.701%); 4687/255300 objects misplaced (1.836%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:37.940 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:37 smithi082 bash[79733]: cluster 2023-10-09T04:39:36.480091+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 12002/255300 objects degraded (4.701%); 4687/255300 objects misplaced (1.836%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:37.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:37.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:38.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:37 smithi153 bash[59565]: cluster 2023-10-09T04:39:36.480091+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 12002/255300 objects degraded (4.701%); 4687/255300 objects misplaced (1.836%); 858 KiB/s, 13 objects/s recovering 2023-10-09T04:39:38.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:38.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:38.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:38.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:38.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:38.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:38.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:38.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:38.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:38.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:38.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:38.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:39.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:39.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:39.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:39.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:39.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:39.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:39.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:39.803 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:39] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:39:39.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.840 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:39.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:39.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:39.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:39.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:40.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:40.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:39 smithi153 bash[59565]: cluster 2023-10-09T04:39:38.480765+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12120/248121 objects degraded (4.885%); 4483/248121 objects misplaced (1.807%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:40.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:39 smithi153 bash[59565]: cluster 2023-10-09T04:39:39.682565+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 12120/248121 objects degraded (4.885%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:40.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:40.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:39 smithi082 bash[77063]: cluster 2023-10-09T04:39:38.480765+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12120/248121 objects degraded (4.885%); 4483/248121 objects misplaced (1.807%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:40.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:39 smithi082 bash[77063]: cluster 2023-10-09T04:39:39.682565+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 12120/248121 objects degraded (4.885%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:39 smithi082 bash[79733]: cluster 2023-10-09T04:39:38.480765+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12120/248121 objects degraded (4.885%); 4483/248121 objects misplaced (1.807%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:39 smithi082 bash[79733]: cluster 2023-10-09T04:39:39.682565+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: Degraded data redundancy: 12120/248121 objects degraded (4.885%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:40.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:40.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:40.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:40.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:40.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:40.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:40.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:40.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:40.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:40.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:40.780 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:40.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:40.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:40.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:41.036 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:41.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:41.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:41.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:41.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:41.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:41.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:41.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:41.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:41.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:41.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:41.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:41.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:41.831 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:41.839 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:41 smithi153 bash[59565]: cluster 2023-10-09T04:39:40.481510+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 12120/245721 objects degraded (4.932%); 4483/245721 objects misplaced (1.824%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:41.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:41.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:41.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:41 smithi082 bash[77063]: cluster 2023-10-09T04:39:40.481510+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 12120/245721 objects degraded (4.932%); 4483/245721 objects misplaced (1.824%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:41.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:41 smithi082 bash[79733]: cluster 2023-10-09T04:39:40.481510+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 12120/245721 objects degraded (4.932%); 4483/245721 objects misplaced (1.824%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:41.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:41.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:42.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:42.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:42.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:42.218 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:42.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:42.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:42.370 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:42.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:42.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:42.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:42.639 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:42.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:42.643 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:39:42.705 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:39:42.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:42.757 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:39:42.798 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:39:42.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:42.835 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:39:42.853 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:39:42.881 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:39:42.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[77063]: audit 2023-10-09T04:39:42.283995+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.82:0/80204883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:42.897 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[77063]: audit 2023-10-09T04:39:42.284738+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:42.898 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[77063]: audit 2023-10-09T04:39:42.522663+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.82:0/3651314263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:42.898 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[79733]: audit 2023-10-09T04:39:42.283995+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.82:0/80204883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:42.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[79733]: audit 2023-10-09T04:39:42.284738+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:42.899 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:42 smithi082 bash[79733]: audit 2023-10-09T04:39:42.522663+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.82:0/3651314263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:42.925 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:39:42.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:42.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:42.959 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:39:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:42.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.970 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.971 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.971 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:39:42.971 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:39:43.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:43.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:42 smithi153 bash[59565]: audit 2023-10-09T04:39:42.283995+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.82:0/80204883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:42 smithi153 bash[59565]: audit 2023-10-09T04:39:42.284738+0000 mon.a (mon.0) 1104 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:43.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:42 smithi153 bash[59565]: audit 2023-10-09T04:39:42.522663+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.82:0/3651314263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:43.193 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:43.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:43.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:43.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:43.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:43.483 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:43.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:43.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:43.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:43.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:43.853 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:43.856 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 7 pg_num to 26 2023-10-09T04:39:43.862 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.482082+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12120/243465 objects degraded (4.978%); 4483/243465 objects misplaced (1.841%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:43.862 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.819938+0000 mon.a (mon.0) 1105 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.836424+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: audit 2023-10-09T04:39:42.846170+0000 mon.a (mon.0) 1106 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.846285+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.847680+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[77063]: audit 2023-10-09T04:39:42.926644+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.482082+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12120/243465 objects degraded (4.978%); 4483/243465 objects misplaced (1.841%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.819938+0000 mon.a (mon.0) 1105 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.836424+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: audit 2023-10-09T04:39:42.846170+0000 mon.a (mon.0) 1106 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:43.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.846285+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:39:43.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.847680+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:43.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:43 smithi082 bash[79733]: audit 2023-10-09T04:39:42.926644+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:39:43.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:43.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:43.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:44.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:44.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:44.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.482082+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12120/243465 objects degraded (4.978%); 4483/243465 objects misplaced (1.841%); 917 KiB/s, 14 objects/s recovering 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.819938+0000 mon.a (mon.0) 1105 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.836424+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: audit 2023-10-09T04:39:42.846170+0000 mon.a (mon.0) 1106 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.846285+0000 mon.a (mon.0) 1107 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.847680+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:43 smithi153 bash[59565]: audit 2023-10-09T04:39:42.926644+0000 mon.a (mon.0) 1108 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:39:44.252 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:44.287 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:44.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:44.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:44.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:44.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:44.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:44.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:44.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:44.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:44.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:44.946 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:39:44.946 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:39:44.968 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:44.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:44.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.872512+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:42.874881+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:43.842246+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:43.843476+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: audit 2023-10-09T04:39:43.845191+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:43.845315+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:43.847488+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: audit 2023-10-09T04:39:44.282384+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: audit 2023-10-09T04:39:44.283855+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: audit 2023-10-09T04:39:44.484263+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[77063]: cluster 2023-10-09T04:39:44.683358+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: Degraded data redundancy: 12120/243465 objects degraded (4.978%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:44.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.872512+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:42.874881+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:43.842246+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:43.843476+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: audit 2023-10-09T04:39:43.845191+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:43.845315+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:43.847488+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: audit 2023-10-09T04:39:44.282384+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: audit 2023-10-09T04:39:44.283855+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: audit 2023-10-09T04:39:44.484263+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:39:44.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:44 smithi082 bash[79733]: cluster 2023-10-09T04:39:44.683358+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: Degraded data redundancy: 12120/243465 objects degraded (4.978%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:45.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:45.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.872512+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:45.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:42.874881+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T04:39:45.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:43.842246+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:43.843476+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: audit 2023-10-09T04:39:43.845191+0000 mon.a (mon.0) 1109 : audit 1 from='client.? 172.21.15.82:0/2642076192' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:43.845315+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:43.847488+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: audit 2023-10-09T04:39:44.282384+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: audit 2023-10-09T04:39:44.283855+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: audit 2023-10-09T04:39:44.484263+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:39:45.080 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:44 smithi153 bash[59565]: cluster 2023-10-09T04:39:44.683358+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: Degraded data redundancy: 12120/243465 objects degraded (4.978%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:45.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:45.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:45.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:45.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:45.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:45.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:45.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:45.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:45.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:45.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:45.838 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:45.941 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:45.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:45.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:46.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:46.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: cluster 2023-10-09T04:39:44.482708+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 12234/237903 objects degraded (5.142%); 4290/237903 objects misplaced (1.803%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:46.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: cluster 2023-10-09T04:39:45.286283+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: audit 2023-10-09T04:39:45.288675+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:39:46.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: cluster 2023-10-09T04:39:45.288841+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:39:46.082 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: cluster 2023-10-09T04:39:45.294330+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.083 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:45 smithi153 bash[59565]: cluster 2023-10-09T04:39:45.321589+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:46.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: cluster 2023-10-09T04:39:44.482708+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 12234/237903 objects degraded (5.142%); 4290/237903 objects misplaced (1.803%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: cluster 2023-10-09T04:39:45.286283+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: audit 2023-10-09T04:39:45.288675+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: cluster 2023-10-09T04:39:45.288841+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: cluster 2023-10-09T04:39:45.294330+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[77063]: cluster 2023-10-09T04:39:45.321589+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: cluster 2023-10-09T04:39:44.482708+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 12234/237903 objects degraded (5.142%); 4290/237903 objects misplaced (1.803%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:39:46.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: cluster 2023-10-09T04:39:45.286283+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: audit 2023-10-09T04:39:45.288675+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:39:46.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: cluster 2023-10-09T04:39:45.288841+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:39:46.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: cluster 2023-10-09T04:39:45.294330+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:45 smithi082 bash[79733]: cluster 2023-10-09T04:39:45.321589+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.235 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:46.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:46.375 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:46.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:46.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:46.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:46.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:46.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:46.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:46.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:46.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:46.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:46.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:46.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:45.316277+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.302340+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.305303+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.306220+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.309421+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.310903+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.312043+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.312560+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.314519+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.315880+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:45.316277+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.302340+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.305303+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.306220+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.309421+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.310903+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.312043+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.312560+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.314519+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:46 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.315880+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:46.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:47.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:47.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:47.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:45.316277+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T04:39:47.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.302340+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.305303+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.306220+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.309421+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.310903+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.312043+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.312560+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.314519+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:46 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.315880+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:47.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:47.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:47.633 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:47.664 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:47.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:47.911 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.920 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.306295+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.920 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.309503+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:47.920 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:46.483376+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v775: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12234/229326 objects degraded (5.335%); 4290/229326 objects misplaced (1.871%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.038196+0000 osd.7 (osd.7) 116 : cluster 0 7.5 deep-scrub starts 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.083759+0000 osd.2 (osd.2) 2 : cluster 0 7.8 deep-scrub starts 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.307929+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.309484+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.318356+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.318400+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:47.921 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:47 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.322405+0000 osd.5 (osd.5) 51 : cluster 0 7.0 scrub starts 2023-10-09T04:39:47.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:47.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:48.054 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:48.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.306295+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:48.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.309503+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:48.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:46.483376+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v775: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12234/229326 objects degraded (5.335%); 4290/229326 objects misplaced (1.871%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:48.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.038196+0000 osd.7 (osd.7) 116 : cluster 0 7.5 deep-scrub starts 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.083759+0000 osd.2 (osd.2) 2 : cluster 0 7.8 deep-scrub starts 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.307929+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.309484+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.318356+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.318400+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.322405+0000 osd.5 (osd.5) 51 : cluster 0 7.0 scrub starts 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.306295+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.309503+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T04:39:48.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:46.483376+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v775: 99 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12234/229326 objects degraded (5.335%); 4290/229326 objects misplaced (1.871%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.038196+0000 osd.7 (osd.7) 116 : cluster 0 7.5 deep-scrub starts 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.083759+0000 osd.2 (osd.2) 2 : cluster 0 7.8 deep-scrub starts 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.307929+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.309484+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.318356+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.318400+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:47 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.322405+0000 osd.5 (osd.5) 51 : cluster 0 7.0 scrub starts 2023-10-09T04:39:48.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:48.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:48.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:48.355 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:48.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:39:48.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:48.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:48.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:48.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:48.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:48.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:48.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:48.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:48.912 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:48.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:48.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.310716+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[77063]: cluster 2023-10-09T04:39:47.313396+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[77063]: cluster 2023-10-09T04:39:48.363009+0000 osd.5 (osd.5) 52 : cluster 0 7.0 scrub starts 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.310716+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[79733]: cluster 2023-10-09T04:39:47.313396+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:48.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:48 smithi082 bash[79733]: cluster 2023-10-09T04:39:48.363009+0000 osd.5 (osd.5) 52 : cluster 0 7.0 scrub starts 2023-10-09T04:39:48.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:39:49.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:49.072 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:49.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:48 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.310716+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:49.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:48 smithi153 bash[59565]: cluster 2023-10-09T04:39:47.313396+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T04:39:49.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:48 smithi153 bash[59565]: cluster 2023-10-09T04:39:48.363009+0000 osd.5 (osd.5) 52 : cluster 0 7.0 scrub starts 2023-10-09T04:39:49.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:49.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:49.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:49.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:49.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:49.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:49.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:49.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:49.725 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:49.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:49] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:39:49.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:49.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:49.901 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:49.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:50.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:50.013 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[77063]: cluster 2023-10-09T04:39:48.484154+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v777: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 3 unknown, 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/224841 objects degraded (5.998%); 1986/224841 objects misplaced (0.883%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T04:39:50.013 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[77063]: cluster 2023-10-09T04:39:49.375612+0000 osd.5 (osd.5) 53 : cluster 0 7.0 scrub starts 2023-10-09T04:39:50.013 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[77063]: cluster 2023-10-09T04:39:49.684520+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 13486/224841 objects degraded (5.998%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:50.014 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[79733]: cluster 2023-10-09T04:39:48.484154+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v777: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 3 unknown, 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/224841 objects degraded (5.998%); 1986/224841 objects misplaced (0.883%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T04:39:50.014 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[79733]: cluster 2023-10-09T04:39:49.375612+0000 osd.5 (osd.5) 53 : cluster 0 7.0 scrub starts 2023-10-09T04:39:50.014 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:49 smithi082 bash[79733]: cluster 2023-10-09T04:39:49.684520+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 13486/224841 objects degraded (5.998%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:50.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:50.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:39:50.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:49 smithi153 bash[59565]: cluster 2023-10-09T04:39:48.484154+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v777: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 3 unknown, 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/224841 objects degraded (5.998%); 1986/224841 objects misplaced (0.883%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T04:39:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:49 smithi153 bash[59565]: cluster 2023-10-09T04:39:49.375612+0000 osd.5 (osd.5) 53 : cluster 0 7.0 scrub starts 2023-10-09T04:39:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:49 smithi153 bash[59565]: cluster 2023-10-09T04:39:49.684520+0000 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 13486/224841 objects degraded (5.998%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:50.162 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:50.266 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:50.305 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:50.411 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:50.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:50.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:50.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:50.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:50.811 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:50.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.951 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:50 smithi153 bash[59565]: cluster 2023-10-09T04:39:50.329565+0000 osd.5 (osd.5) 54 : cluster 0 7.0 scrub starts 2023-10-09T04:39:50.982 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:50.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:51.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:51.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:50 smithi082 bash[77063]: cluster 2023-10-09T04:39:50.329565+0000 osd.5 (osd.5) 54 : cluster 0 7.0 scrub starts 2023-10-09T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:50 smithi082 bash[79733]: cluster 2023-10-09T04:39:50.329565+0000 osd.5 (osd.5) 54 : cluster 0 7.0 scrub starts 2023-10-09T04:39:51.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:51.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:51.378 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:51.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:51.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:51.610 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:51.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:51.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:51.872 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:51 smithi153 bash[59565]: cluster 2023-10-09T04:39:50.484802+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v778: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 1 unknown, 91 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/223683 objects degraded (6.029%); 1986/223683 objects misplaced (0.888%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:39:51.877 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:51.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:51.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:52.016 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:52.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:52.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:51 smithi082 bash[77063]: cluster 2023-10-09T04:39:50.484802+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v778: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 1 unknown, 91 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/223683 objects degraded (6.029%); 1986/223683 objects misplaced (0.888%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:39:52.158 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:51 smithi082 bash[79733]: cluster 2023-10-09T04:39:50.484802+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v778: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 1 peering, 1 unknown, 91 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13486/223683 objects degraded (6.029%); 1986/223683 objects misplaced (0.888%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:39:52.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:52.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:52.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:52.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:52.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:52.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:52.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:52.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:52.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:52.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:52.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:52.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:52.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:53.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:53.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:53.105 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:53.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:53.155 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:39:53.155 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:39:53.155 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:39:53.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T04:39:53.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:53.284 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:53.284 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:53.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:53.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:53.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:53.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:53.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:53.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:53.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:53.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:53.916 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:53 smithi153 bash[59565]: cluster 2023-10-09T04:39:52.485465+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v779: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 13486/228231 objects degraded (5.909%); 1986/228231 objects misplaced (0.870%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:39:53.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:53.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:54.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:54.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:54.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:53 smithi082 bash[77063]: cluster 2023-10-09T04:39:52.485465+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v779: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 13486/228231 objects degraded (5.909%); 1986/228231 objects misplaced (0.870%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:39:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:53 smithi082 bash[79733]: cluster 2023-10-09T04:39:52.485465+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v779: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+degraded, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 13486/228231 objects degraded (5.909%); 1986/228231 objects misplaced (0.870%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:39:54.305 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:54.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:54.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:54.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:54.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:54.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:54.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:54.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:54.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:54.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:54.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:55.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:55.120 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.130 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:54 smithi153 bash[59565]: cluster 2023-10-09T04:39:54.685773+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 13486/228231 objects degraded (5.909%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:55.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:54 smithi082 bash[77063]: cluster 2023-10-09T04:39:54.685773+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 13486/228231 objects degraded (5.909%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:55.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:54 smithi082 bash[79733]: cluster 2023-10-09T04:39:54.685773+0000 mon.a (mon.0) 1120 : cluster 3 Health check update: Degraded data redundancy: 13486/228231 objects degraded (5.909%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:55.252 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:55.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:55.385 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:55.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:55.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:55.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:55.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.683 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:55.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.813 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:55.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:55 smithi153 bash[59565]: cluster 2023-10-09T04:39:54.486331+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v780: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 MiB/s wr, 1.10k op/s; 12980/221835 objects degraded (5.851%); 1730/221835 objects misplaced (0.780%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:39:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:55 smithi153 bash[59565]: cluster 2023-10-09T04:39:55.362358+0000 osd.5 (osd.5) 55 : cluster 0 7.0 scrub starts 2023-10-09T04:39:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:55 smithi153 bash[59565]: audit 2023-10-09T04:39:55.880882+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:55.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:55.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:56.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:56.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:56.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:56.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[77063]: cluster 2023-10-09T04:39:54.486331+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v780: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 MiB/s wr, 1.10k op/s; 12980/221835 objects degraded (5.851%); 1730/221835 objects misplaced (0.780%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[77063]: cluster 2023-10-09T04:39:55.362358+0000 osd.5 (osd.5) 55 : cluster 0 7.0 scrub starts 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[77063]: audit 2023-10-09T04:39:55.880882+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[79733]: cluster 2023-10-09T04:39:54.486331+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v780: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 25 MiB/s wr, 1.10k op/s; 12980/221835 objects degraded (5.851%); 1730/221835 objects misplaced (0.780%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[79733]: cluster 2023-10-09T04:39:55.362358+0000 osd.5 (osd.5) 55 : cluster 0 7.0 scrub starts 2023-10-09T04:39:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:55 smithi082 bash[79733]: audit 2023-10-09T04:39:55.880882+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:39:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:56.334 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:56.357 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:56.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:56.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:56.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:56.595 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:56.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:56.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:56.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:56.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:56.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:56.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:57.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:57.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:56 smithi082 bash[77063]: cluster 2023-10-09T04:39:56.352420+0000 osd.5 (osd.5) 56 : cluster 0 7.0 scrub starts 2023-10-09T04:39:57.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:56 smithi082 bash[79733]: cluster 2023-10-09T04:39:56.352420+0000 osd.5 (osd.5) 56 : cluster 0 7.0 scrub starts 2023-10-09T04:39:57.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:57.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:57.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:56 smithi153 bash[59565]: cluster 2023-10-09T04:39:56.352420+0000 osd.5 (osd.5) 56 : cluster 0 7.0 scrub starts 2023-10-09T04:39:57.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.293 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:57.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:57.356 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:57.394 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:57.424 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:57.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:57.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:57.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:57.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:57.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:57.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:57.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:57.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:58.067 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:58.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:57 smithi153 bash[59565]: cluster 2023-10-09T04:39:56.487049+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v781: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 20 MiB/s wr, 945 op/s; 12980/220521 objects degraded (5.886%); 1730/220521 objects misplaced (0.785%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:39:58.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:58.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:58.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:58.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:57 smithi082 bash[77063]: cluster 2023-10-09T04:39:56.487049+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v781: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 20 MiB/s wr, 945 op/s; 12980/220521 objects degraded (5.886%); 1730/220521 objects misplaced (0.785%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:39:58.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:57 smithi082 bash[79733]: cluster 2023-10-09T04:39:56.487049+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v781: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 20 MiB/s wr, 945 op/s; 12980/220521 objects degraded (5.886%); 1730/220521 objects misplaced (0.785%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:39:58.324 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.356 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:58.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.494 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:58.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:58.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:58.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.766 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:58.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:58.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:58.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:59.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:58 smithi153 bash[59565]: cluster 2023-10-09T04:39:58.346037+0000 osd.5 (osd.5) 57 : cluster 0 7.0 scrub starts 2023-10-09T04:39:59.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:59.141 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:58 smithi082 bash[77063]: cluster 2023-10-09T04:39:58.346037+0000 osd.5 (osd.5) 57 : cluster 0 7.0 scrub starts 2023-10-09T04:39:59.141 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:58 smithi082 bash[79733]: cluster 2023-10-09T04:39:58.346037+0000 osd.5 (osd.5) 57 : cluster 0 7.0 scrub starts 2023-10-09T04:39:59.164 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:59.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:59.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:59.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:59.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:59.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:59.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:59.606 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:59.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:59.852 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:39:59.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:59.944 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[77063]: cluster 2023-10-09T04:39:58.487732+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v782: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 1.02k op/s; 12428/213159 objects degraded (5.830%); 1434/213159 objects misplaced (0.673%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[77063]: audit 2023-10-09T04:39:59.278103+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[77063]: cluster 2023-10-09T04:39:59.686836+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:39:59] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[79733]: cluster 2023-10-09T04:39:58.487732+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v782: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 1.02k op/s; 12428/213159 objects degraded (5.830%); 1434/213159 objects misplaced (0.673%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[79733]: audit 2023-10-09T04:39:59.278103+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:59.945 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:39:59 smithi082 bash[79733]: cluster 2023-10-09T04:39:59.686836+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:59.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:00.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:00.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:00.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:59 smithi153 bash[59565]: cluster 2023-10-09T04:39:58.487732+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v782: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 18 MiB/s wr, 1.02k op/s; 12428/213159 objects degraded (5.830%); 1434/213159 objects misplaced (0.673%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:40:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:59 smithi153 bash[59565]: audit 2023-10-09T04:39:59.278103+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:39:59 smithi153 bash[59565]: cluster 2023-10-09T04:39:59.686836+0000 mon.a (mon.0) 1123 : cluster 3 Health check update: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:00.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:00.302 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:00.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:00.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:00.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:00.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:00.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:00.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:00.701 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:00.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:00.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:00.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:00.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:00.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000152+0000 mon.a (mon.0) 1124 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000191+0000 mon.a (mon.0) 1125 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000204+0000 mon.a (mon.0) 1126 : cluster 3 pg 7.1 is active+recovery_wait+degraded, acting [6,3,1] 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000250+0000 mon.a (mon.0) 1127 : cluster 3 pg 7.4 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000262+0000 mon.a (mon.0) 1128 : cluster 3 pg 7.9 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000277+0000 mon.a (mon.0) 1129 : cluster 3 pg 7.14 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:00.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.000292+0000 mon.a (mon.0) 1130 : cluster 3 pg 7.19 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:00.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000152+0000 mon.a (mon.0) 1124 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:00.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000191+0000 mon.a (mon.0) 1125 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:00.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000204+0000 mon.a (mon.0) 1126 : cluster 3 pg 7.1 is active+recovery_wait+degraded, acting [6,3,1] 2023-10-09T04:40:00.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000250+0000 mon.a (mon.0) 1127 : cluster 3 pg 7.4 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:00.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000262+0000 mon.a (mon.0) 1128 : cluster 3 pg 7.9 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:00.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000277+0000 mon.a (mon.0) 1129 : cluster 3 pg 7.14 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:00.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:00 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.000292+0000 mon.a (mon.0) 1130 : cluster 3 pg 7.19 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:01.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:01.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:01.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:01.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:01.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:01.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:01.399 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000152+0000 mon.a (mon.0) 1124 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:01.399 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000191+0000 mon.a (mon.0) 1125 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12428/213159 objects degraded (5.830%), 5 pgs degraded 2023-10-09T04:40:01.399 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000204+0000 mon.a (mon.0) 1126 : cluster 3 pg 7.1 is active+recovery_wait+degraded, acting [6,3,1] 2023-10-09T04:40:01.400 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000250+0000 mon.a (mon.0) 1127 : cluster 3 pg 7.4 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:01.400 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000262+0000 mon.a (mon.0) 1128 : cluster 3 pg 7.9 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:01.400 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000277+0000 mon.a (mon.0) 1129 : cluster 3 pg 7.14 is active+recovery_wait+degraded, acting [0,1,6] 2023-10-09T04:40:01.400 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:00 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.000292+0000 mon.a (mon.0) 1130 : cluster 3 pg 7.19 is active+recovery_wait+degraded, acting [0,1,7] 2023-10-09T04:40:01.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:01.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:01.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:01.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:01.689 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:01.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:01.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:01.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:01.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:01.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:01.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:01.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:02.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:02.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:02.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:02.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:01 smithi082 bash[77063]: cluster 2023-10-09T04:40:00.488390+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v783: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 MiB/s wr, 916 op/s; 12428/211404 objects degraded (5.879%); 1434/211404 objects misplaced (0.678%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:02.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:01 smithi082 bash[79733]: cluster 2023-10-09T04:40:00.488390+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v783: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 MiB/s wr, 916 op/s; 12428/211404 objects degraded (5.879%); 1434/211404 objects misplaced (0.678%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:02.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:02.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:02.378 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:01 smithi153 bash[59565]: cluster 2023-10-09T04:40:00.488390+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v783: 99 pgs: 1 active+recovering, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 MiB/s wr, 916 op/s; 12428/211404 objects degraded (5.879%); 1434/211404 objects misplaced (0.678%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:02.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:02.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:02.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:02.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:02.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:02.891 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:02.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:02.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:02.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:03.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:03.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[77063]: cluster 2023-10-09T04:40:01.572237+0000 osd.2 (osd.2) 3 : cluster 0 7.8 deep-scrub ok 2023-10-09T04:40:03.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[77063]: audit 2023-10-09T04:40:02.340488+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[77063]: audit 2023-10-09T04:40:02.350166+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.031 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[79733]: cluster 2023-10-09T04:40:01.572237+0000 osd.2 (osd.2) 3 : cluster 0 7.8 deep-scrub ok 2023-10-09T04:40:03.031 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[79733]: audit 2023-10-09T04:40:02.340488+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.031 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:02 smithi082 bash[79733]: audit 2023-10-09T04:40:02.350166+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:03.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:02 smithi153 bash[59565]: cluster 2023-10-09T04:40:01.572237+0000 osd.2 (osd.2) 3 : cluster 0 7.8 deep-scrub ok 2023-10-09T04:40:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:02 smithi153 bash[59565]: audit 2023-10-09T04:40:02.340488+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:02 smithi153 bash[59565]: audit 2023-10-09T04:40:02.350166+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:03.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:03.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:03.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:03.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:03.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:03.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:03.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:03.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:03.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:03.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:03.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:04.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:04.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[77063]: cluster 2023-10-09T04:40:02.489076+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v784: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12428/208317 objects degraded (5.966%); 1434/208317 objects misplaced (0.688%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:04.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[77063]: cluster 2023-10-09T04:40:03.208836+0000 osd.7 (osd.7) 118 : cluster 0 7.5 scrub starts 2023-10-09T04:40:04.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[77063]: cluster 2023-10-09T04:40:03.297761+0000 osd.5 (osd.5) 58 : cluster 0 7.0 scrub starts 2023-10-09T04:40:04.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[79733]: cluster 2023-10-09T04:40:02.489076+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v784: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12428/208317 objects degraded (5.966%); 1434/208317 objects misplaced (0.688%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:04.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[79733]: cluster 2023-10-09T04:40:03.208836+0000 osd.7 (osd.7) 118 : cluster 0 7.5 scrub starts 2023-10-09T04:40:04.094 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:03 smithi082 bash[79733]: cluster 2023-10-09T04:40:03.297761+0000 osd.5 (osd.5) 58 : cluster 0 7.0 scrub starts 2023-10-09T04:40:04.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:04.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:04.221 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:03 smithi153 bash[59565]: cluster 2023-10-09T04:40:02.489076+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v784: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12428/208317 objects degraded (5.966%); 1434/208317 objects misplaced (0.688%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:40:04.221 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:03 smithi153 bash[59565]: cluster 2023-10-09T04:40:03.208836+0000 osd.7 (osd.7) 118 : cluster 0 7.5 scrub starts 2023-10-09T04:40:04.221 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:03 smithi153 bash[59565]: cluster 2023-10-09T04:40:03.297761+0000 osd.5 (osd.5) 58 : cluster 0 7.0 scrub starts 2023-10-09T04:40:04.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:04.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:04.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:04.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:04.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:04.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:04.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:04.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:04.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:04.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:04.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:04.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:04.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:04.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:05.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:05.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:05.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[77063]: cluster 2023-10-09T04:40:04.160212+0000 osd.7 (osd.7) 119 : cluster 0 7.5 scrub starts 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[77063]: cluster 2023-10-09T04:40:04.253800+0000 osd.5 (osd.5) 59 : cluster 0 7.0 scrub starts 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[77063]: cluster 2023-10-09T04:40:04.688098+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 12428/208317 objects degraded (5.966%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[79733]: cluster 2023-10-09T04:40:04.160212+0000 osd.7 (osd.7) 119 : cluster 0 7.5 scrub starts 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[79733]: cluster 2023-10-09T04:40:04.253800+0000 osd.5 (osd.5) 59 : cluster 0 7.0 scrub starts 2023-10-09T04:40:05.176 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:04 smithi082 bash[79733]: cluster 2023-10-09T04:40:04.688098+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 12428/208317 objects degraded (5.966%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:05.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:05.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:05.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:04 smithi153 bash[59565]: cluster 2023-10-09T04:40:04.160212+0000 osd.7 (osd.7) 119 : cluster 0 7.5 scrub starts 2023-10-09T04:40:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:04 smithi153 bash[59565]: cluster 2023-10-09T04:40:04.253800+0000 osd.5 (osd.5) 59 : cluster 0 7.0 scrub starts 2023-10-09T04:40:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:04 smithi153 bash[59565]: cluster 2023-10-09T04:40:04.688098+0000 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 12428/208317 objects degraded (5.966%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:05.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:05.462 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:05.567 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:05.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:05.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:05.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:05.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:05.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:05.863 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:05.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:05.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:05.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:05.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:06.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:06.003 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:05 smithi082 bash[77063]: cluster 2023-10-09T04:40:04.489830+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v785: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 11901/202107 objects degraded (5.888%); 1140/202107 objects misplaced (0.564%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:40:06.003 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:05 smithi082 bash[79733]: cluster 2023-10-09T04:40:04.489830+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v785: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 11901/202107 objects degraded (5.888%); 1140/202107 objects misplaced (0.564%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:40:06.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:06.134 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:06.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:05 smithi153 bash[59565]: cluster 2023-10-09T04:40:04.489830+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v785: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 11901/202107 objects degraded (5.888%); 1140/202107 objects misplaced (0.564%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:40:06.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:06.266 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:06.370 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:06.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:06.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:06.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:06.529 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:06.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:06.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:06.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:06.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:06.905 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:06.933 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:06.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:06.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:07.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:07.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:06 smithi153 bash[59565]: cluster 2023-10-09T04:40:06.252550+0000 osd.5 (osd.5) 60 : cluster 0 7.0 scrub starts 2023-10-09T04:40:07.177 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:07.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:06 smithi082 bash[77063]: cluster 2023-10-09T04:40:06.252550+0000 osd.5 (osd.5) 60 : cluster 0 7.0 scrub starts 2023-10-09T04:40:07.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:06 smithi082 bash[79733]: cluster 2023-10-09T04:40:06.252550+0000 osd.5 (osd.5) 60 : cluster 0 7.0 scrub starts 2023-10-09T04:40:07.306 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:07.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:07.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:07.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:07.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:07.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:07.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:07.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:07.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:07.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:08.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:08.011 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:07 smithi153 bash[59565]: cluster 2023-10-09T04:40:06.490470+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v786: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 11901/200745 objects degraded (5.928%); 1140/200745 objects misplaced (0.568%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:40:08.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:07 smithi153 bash[59565]: cluster 2023-10-09T04:40:07.044651+0000 osd.2 (osd.2) 4 : cluster 0 7.3 scrub starts 2023-10-09T04:40:08.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:07 smithi153 bash[59565]: cluster 2023-10-09T04:40:07.186658+0000 osd.7 (osd.7) 120 : cluster 0 7.5 scrub starts 2023-10-09T04:40:08.012 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:07 smithi153 bash[59565]: cluster 2023-10-09T04:40:07.233394+0000 osd.5 (osd.5) 61 : cluster 0 7.0 scrub starts 2023-10-09T04:40:08.032 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:08.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:08.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:08.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[77063]: cluster 2023-10-09T04:40:06.490470+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v786: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 11901/200745 objects degraded (5.928%); 1140/200745 objects misplaced (0.568%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:40:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[77063]: cluster 2023-10-09T04:40:07.044651+0000 osd.2 (osd.2) 4 : cluster 0 7.3 scrub starts 2023-10-09T04:40:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[77063]: cluster 2023-10-09T04:40:07.186658+0000 osd.7 (osd.7) 120 : cluster 0 7.5 scrub starts 2023-10-09T04:40:08.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[77063]: cluster 2023-10-09T04:40:07.233394+0000 osd.5 (osd.5) 61 : cluster 0 7.0 scrub starts 2023-10-09T04:40:08.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[79733]: cluster 2023-10-09T04:40:06.490470+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v786: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 11901/200745 objects degraded (5.928%); 1140/200745 objects misplaced (0.568%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:40:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[79733]: cluster 2023-10-09T04:40:07.044651+0000 osd.2 (osd.2) 4 : cluster 0 7.3 scrub starts 2023-10-09T04:40:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[79733]: cluster 2023-10-09T04:40:07.186658+0000 osd.7 (osd.7) 120 : cluster 0 7.5 scrub starts 2023-10-09T04:40:08.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:07 smithi082 bash[79733]: cluster 2023-10-09T04:40:07.233394+0000 osd.5 (osd.5) 61 : cluster 0 7.0 scrub starts 2023-10-09T04:40:08.265 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:08.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:08.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:08.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:08.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:08.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:08.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:08.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:08.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:08.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:08.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:09.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:09.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:09.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:08 smithi082 bash[77063]: cluster 2023-10-09T04:40:08.006149+0000 osd.2 (osd.2) 5 : cluster 0 7.13 scrub starts 2023-10-09T04:40:09.105 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:08 smithi082 bash[79733]: cluster 2023-10-09T04:40:08.006149+0000 osd.2 (osd.2) 5 : cluster 0 7.13 scrub starts 2023-10-09T04:40:09.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:09.212 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:08 smithi153 bash[59565]: cluster 2023-10-09T04:40:08.006149+0000 osd.2 (osd.2) 5 : cluster 0 7.13 scrub starts 2023-10-09T04:40:09.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:09.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:09.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:09.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.477 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:40:09.478 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:40:09.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:09.543 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:40:09.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.602 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:40:09.625 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:09.636 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:40:09.688 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:40:09.724 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:09] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-09T04:40:09.724 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:40:09.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:09.763 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:40:09.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:09.823 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:40:09.866 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:40:09.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.878 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.879 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:40:09.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:09.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:09.979 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: cluster 2023-10-09T04:40:08.491173+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v787: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 11311/191652 objects degraded (5.902%); 854/191652 objects misplaced (0.446%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:40:09.979 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: cluster 2023-10-09T04:40:08.491173+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v787: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 11311/191652 objects degraded (5.902%); 854/191652 objects misplaced (0.446%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:40:09.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.133576+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:09.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.142759+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:09.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: cluster 2023-10-09T04:40:09.186394+0000 osd.7 (osd.7) 121 : cluster 0 7.5 scrub starts 2023-10-09T04:40:09.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: cluster 2023-10-09T04:40:09.240438+0000 osd.5 (osd.5) 62 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:40:09.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:10.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:10.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: cluster 2023-10-09T04:40:08.491173+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v787: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 11311/191652 objects degraded (5.902%); 854/191652 objects misplaced (0.446%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.133576+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.142759+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: cluster 2023-10-09T04:40:09.186394+0000 osd.7 (osd.7) 121 : cluster 0 7.5 scrub starts 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: cluster 2023-10-09T04:40:09.240438+0000 osd.5 (osd.5) 62 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.469432+0000 mon.a (mon.0) 1136 : audit 0 from='client.? 172.21.15.82:0/2774603837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: cluster 2023-10-09T04:40:09.689027+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 11311/191652 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.777595+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.779088+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.787385+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:09 smithi153 bash[59565]: audit 2023-10-09T04:40:09.847679+0000 mon.a (mon.0) 1141 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:40:10.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.133576+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.142759+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: cluster 2023-10-09T04:40:09.186394+0000 osd.7 (osd.7) 121 : cluster 0 7.5 scrub starts 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: cluster 2023-10-09T04:40:09.240438+0000 osd.5 (osd.5) 62 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.469432+0000 mon.a (mon.0) 1136 : audit 0 from='client.? 172.21.15.82:0/2774603837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: cluster 2023-10-09T04:40:09.689027+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 11311/191652 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.777595+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.779088+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.787385+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[77063]: audit 2023-10-09T04:40:09.847679+0000 mon.a (mon.0) 1141 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:40:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: debug 2023-10-09T04:40:10.179+0000 7f3fc98c6700 -1 mon.a@0(leader).osd e188 definitely_dead 0 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.469432+0000 mon.a (mon.0) 1136 : audit 0 from='client.? 172.21.15.82:0/2774603837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: cluster 2023-10-09T04:40:09.689027+0000 mon.a (mon.0) 1137 : cluster 3 Health check update: Degraded data redundancy: 11311/191652 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.777595+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.779088+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.787385+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:10.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:09 smithi082 bash[79733]: audit 2023-10-09T04:40:09.847679+0000 mon.a (mon.0) 1141 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:40:10.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:10.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:10.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:10.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:10.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:10.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:10.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:10.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:10.791 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:40:10.796 INFO:teuthology.orchestra.run.smithi082.stderr:marked out osd.7. 2023-10-09T04:40:10.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:10.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:10.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:11.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:11.059 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:09.990568+0000 osd.2 (osd.2) 6 : cluster 0 7.18 scrub starts 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: audit 2023-10-09T04:40:10.183557+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.82:0/3699267650' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: audit 2023-10-09T04:40:10.184856+0000 mon.a (mon.0) 1142 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.184972+0000 mon.a (mon.0) 1143 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.196177+0000 osd.5 (osd.5) 63 : cluster 0 7.0 scrub starts 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.492177+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v788: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 11311/189534 objects degraded (5.968%); 854/189534 objects misplaced (0.451%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.783821+0000 mon.a (mon.0) 1144 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: audit 2023-10-09T04:40:10.787501+0000 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: audit 2023-10-09T04:40:10.787542+0000 mon.a (mon.0) 1146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.787582+0000 mon.a (mon.0) 1147 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-10-09T04:40:11.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:10 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.788220+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:11.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:11.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:11.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:09.990568+0000 osd.2 (osd.2) 6 : cluster 0 7.18 scrub starts 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: audit 2023-10-09T04:40:10.183557+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.82:0/3699267650' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: audit 2023-10-09T04:40:10.184856+0000 mon.a (mon.0) 1142 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.184972+0000 mon.a (mon.0) 1143 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.196177+0000 osd.5 (osd.5) 63 : cluster 0 7.0 scrub starts 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.492177+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v788: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 11311/189534 objects degraded (5.968%); 854/189534 objects misplaced (0.451%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.783821+0000 mon.a (mon.0) 1144 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: audit 2023-10-09T04:40:10.787501+0000 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:40:11.190 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: audit 2023-10-09T04:40:10.787542+0000 mon.a (mon.0) 1146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.787582+0000 mon.a (mon.0) 1147 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.788220+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:09.990568+0000 osd.2 (osd.2) 6 : cluster 0 7.18 scrub starts 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: audit 2023-10-09T04:40:10.183557+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.82:0/3699267650' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: audit 2023-10-09T04:40:10.184856+0000 mon.a (mon.0) 1142 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.184972+0000 mon.a (mon.0) 1143 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T04:40:11.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.196177+0000 osd.5 (osd.5) 63 : cluster 0 7.0 scrub starts 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.492177+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v788: 99 pgs: 1 active+recovering, 5 active+recovery_wait+degraded, 93 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 11311/189534 objects degraded (5.968%); 854/189534 objects misplaced (0.451%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.783821+0000 mon.a (mon.0) 1144 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: audit 2023-10-09T04:40:10.787501+0000 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.82:0/2538285923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: audit 2023-10-09T04:40:10.787542+0000 mon.a (mon.0) 1146 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.787582+0000 mon.a (mon.0) 1147 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-10-09T04:40:11.192 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:10 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.788220+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:11.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:11.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:11.433 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:11.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:40:11.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:11.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:11.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:11.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:11.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:11.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:11.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:12.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.595867+0000 osd.2 (osd.2) 7 : cluster 0 7.18 scrub ok 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[77063]: cluster 2023-10-09T04:40:10.786187+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.789693+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.790620+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.793987+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.077 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.595867+0000 osd.2 (osd.2) 7 : cluster 0 7.18 scrub ok 2023-10-09T04:40:12.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[79733]: cluster 2023-10-09T04:40:10.786187+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:12.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.789693+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-10-09T04:40:12.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.790620+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:11 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.793987+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:12.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:11 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.595867+0000 osd.2 (osd.2) 7 : cluster 0 7.18 scrub ok 2023-10-09T04:40:12.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:11 smithi153 bash[59565]: cluster 2023-10-09T04:40:10.786187+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T04:40:12.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:11 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.789693+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-10-09T04:40:12.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:11 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.790620+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:11 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.793987+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:12.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:12.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:12.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:12.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:12.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:12.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:12.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:12.734 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:12.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:12.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:12.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:13.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:13.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:13.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:13.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.789043+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.791966+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.792635+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.793675+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.795286+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.795504+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.795591+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:11.798175+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.493012+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v791: 99 pgs: 4 remapped+peering, 1 active+recovering, 5 active+recovery_wait+degraded, 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 60 op/s; 11311/186762 objects degraded (6.056%); 854/186762 objects misplaced (0.457%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.792012+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.792881+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.795042+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:13 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.798225+0000 osd.6 (osd.6) 44 : cluster 0 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 188'9009 2023-10-09T04:40:13.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:13.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:13.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:13.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.789043+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.791966+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.792635+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.793675+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.795286+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.795504+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.795591+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.345 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:11.798175+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.493012+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v791: 99 pgs: 4 remapped+peering, 1 active+recovering, 5 active+recovery_wait+degraded, 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 60 op/s; 11311/186762 objects degraded (6.056%); 854/186762 objects misplaced (0.457%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.792012+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.792881+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.795042+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.798225+0000 osd.6 (osd.6) 44 : cluster 0 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 188'9009 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.789043+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.791966+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.792635+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.346 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.793675+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.795286+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.795504+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.795591+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:11.798175+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.493012+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v791: 99 pgs: 4 remapped+peering, 1 active+recovering, 5 active+recovery_wait+degraded, 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 60 op/s; 11311/186762 objects degraded (6.056%); 854/186762 objects misplaced (0.457%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.792012+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.792881+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.795042+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:13.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:13 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.798225+0000 osd.6 (osd.6) 44 : cluster 0 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 188'9009 2023-10-09T04:40:13.417 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:13.443 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:13.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:13.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:13.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:13.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:13.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:13.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:13.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:13.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:13.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:13.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:13.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:14.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:14.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:14.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:14 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.795171+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:14 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.795651+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:14 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.798559+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:14 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.806041+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:14 smithi153 bash[59565]: cluster 2023-10-09T04:40:12.853150+0000 osd.0 (osd.0) 2 : cluster 0 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 188'8949 2023-10-09T04:40:14.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:14.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:14.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.795171+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.795651+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.798559+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.806041+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[77063]: cluster 2023-10-09T04:40:12.853150+0000 osd.0 (osd.0) 2 : cluster 0 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 188'8949 2023-10-09T04:40:14.357 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.795171+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.358 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.795651+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.358 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.798559+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.358 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.806041+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T04:40:14.358 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:14 smithi082 bash[79733]: cluster 2023-10-09T04:40:12.853150+0000 osd.0 (osd.0) 2 : cluster 0 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 188'8949 2023-10-09T04:40:14.381 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:14.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:14.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:14.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:14.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:14.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:14.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:14.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:14.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:14.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:14.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:14.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:15.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:15.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:15.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.023790+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.024291+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.026427+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.027360+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.028040+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.029773+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.040328+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.042312+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.043246+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: audit 2023-10-09T04:40:14.289010+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: audit 2023-10-09T04:40:14.290287+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.493609+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v794: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/183120 objects degraded (4.100%); 3882/183120 objects misplaced (2.120%); 11 MiB/s, 185 objects/s recovering 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:14.690368+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 11311/186762 objects degraded (6.056%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.028870+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.023790+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.024291+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.300 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.026427+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.027360+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.028040+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.029773+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.040328+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.042312+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.043246+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: audit 2023-10-09T04:40:14.289010+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: audit 2023-10-09T04:40:14.290287+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.493609+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v794: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/183120 objects degraded (4.100%); 3882/183120 objects misplaced (2.120%); 11 MiB/s, 185 objects/s recovering 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:14.690368+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 11311/186762 objects degraded (6.056%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:15.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:15 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.028870+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-10-09T04:40:15.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:15.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:40:15.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.023790+0000 mon.a (mon.0) 1150 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-10-09T04:40:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.024291+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.026427+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.027360+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.028040+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.029773+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.040328+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.042312+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.043246+0000 osd.4 (osd.4) 135 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: audit 2023-10-09T04:40:14.289010+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: audit 2023-10-09T04:40:14.290287+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.493609+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v794: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/183120 objects degraded (4.100%); 3882/183120 objects misplaced (2.120%); 11 MiB/s, 185 objects/s recovering 2023-10-09T04:40:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:14.690368+0000 mon.a (mon.0) 1153 : cluster 3 Health check update: Degraded data redundancy: 11311/186762 objects degraded (6.056%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:15.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:15 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.028870+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-10-09T04:40:15.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:15.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:15.577 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:15.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:15.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:15.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:15.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:15.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:16.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:16.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.027158+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.029969+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.031522+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.040293+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.055705+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[77063]: cluster 2023-10-09T04:40:15.285764+0000 mon.a (mon.0) 1155 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.027158+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.029969+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.031522+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.040293+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.055705+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:16 smithi082 bash[79733]: cluster 2023-10-09T04:40:15.285764+0000 mon.a (mon.0) 1155 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:40:16.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:16.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:16.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.027158+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.029969+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.031522+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.040293+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.055705+0000 osd.4 (osd.4) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T04:40:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:16 smithi153 bash[59565]: cluster 2023-10-09T04:40:15.285764+0000 mon.a (mon.0) 1155 : cluster 3 Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:40:16.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:16.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:16.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:16.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:16.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:16.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:16.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:16.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:40:17.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:17.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:17.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:17.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:17.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:17.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:17 smithi082 bash[77063]: cluster 2023-10-09T04:40:16.494247+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v796: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/182727 objects degraded (4.109%); 3882/182727 objects misplaced (2.124%); 10 MiB/s, 2 keys/s, 161 objects/s recovering 2023-10-09T04:40:17.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:17 smithi082 bash[79733]: cluster 2023-10-09T04:40:16.494247+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v796: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/182727 objects degraded (4.109%); 3882/182727 objects misplaced (2.124%); 10 MiB/s, 2 keys/s, 161 objects/s recovering 2023-10-09T04:40:17.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:17.300 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:17 smithi153 bash[59565]: cluster 2023-10-09T04:40:16.494247+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v796: 99 pgs: 11 remapped+peering, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 5 active+recovery_wait+degraded, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7508/182727 objects degraded (4.109%); 3882/182727 objects misplaced (2.124%); 10 MiB/s, 2 keys/s, 161 objects/s recovering 2023-10-09T04:40:17.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:17.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:17.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:17.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:17.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:17.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:17.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:17.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:17.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:17.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:17.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:17.968 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:17.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:18.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:18.088 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:18.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:18.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:18.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:18.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:18.385 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:18.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:18.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:18.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:18.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:18.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:18.633 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:18 smithi153 bash[59565]: cluster 2023-10-09T04:40:18.546507+0000 mon.a (mon.0) 1156 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:40:18.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:18.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:18.782 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:18.885 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:18.894 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:18 smithi082 bash[79733]: cluster 2023-10-09T04:40:18.546507+0000 mon.a (mon.0) 1156 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:40:18.895 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:18 smithi082 bash[77063]: cluster 2023-10-09T04:40:18.546507+0000 mon.a (mon.0) 1156 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:40:18.914 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:18.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:19.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:19.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:19.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:19.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:19.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:19.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:19.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:19.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:19.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:19.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:19.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:19.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:19.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:19.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:19.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:19 smithi153 bash[59565]: cluster 2023-10-09T04:40:18.495057+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v797: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 31368/174951 objects degraded (17.930%); 8582/174951 objects misplaced (4.905%); 11 MiB/s, 1 keys/s, 179 objects/s recovering 2023-10-09T04:40:19.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:19.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:19 smithi082 bash[77063]: cluster 2023-10-09T04:40:18.495057+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v797: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 31368/174951 objects degraded (17.930%); 8582/174951 objects misplaced (4.905%); 11 MiB/s, 1 keys/s, 179 objects/s recovering 2023-10-09T04:40:19.959 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:19] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:40:19.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:19 smithi082 bash[79733]: cluster 2023-10-09T04:40:18.495057+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v797: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 31368/174951 objects degraded (17.930%); 8582/174951 objects misplaced (4.905%); 11 MiB/s, 1 keys/s, 179 objects/s recovering 2023-10-09T04:40:19.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:19.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:20.060 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:40:20.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:40:20.060 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T04:40:20.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T04:40:20.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:20.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:20.125 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:20.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:20.357 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:20.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:20.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:20.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:20 smithi153 bash[59565]: cluster 2023-10-09T04:40:19.691552+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 31368/174951 objects degraded (17.930%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:20.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.774 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:20 smithi082 bash[77063]: cluster 2023-10-09T04:40:19.691552+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 31368/174951 objects degraded (17.930%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:20.774 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:20 smithi082 bash[79733]: cluster 2023-10-09T04:40:19.691552+0000 mon.a (mon.0) 1157 : cluster 3 Health check update: Degraded data redundancy: 31368/174951 objects degraded (17.930%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:20.792 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:20.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:20.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:20.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:20.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:21.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:21.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:21.139 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:21.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:21.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:21.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:21.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:21.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:21.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:21.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:21.600 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:21.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:21.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:21.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:21.860 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:21.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:21.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:21.973 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:21 smithi082 bash[77063]: cluster 2023-10-09T04:40:20.495781+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 31368/172728 objects degraded (18.160%); 8582/172728 objects misplaced (4.969%); 6.5 MiB/s, 1 keys/s, 101 objects/s recovering 2023-10-09T04:40:21.973 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:21 smithi082 bash[79733]: cluster 2023-10-09T04:40:20.495781+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 31368/172728 objects degraded (18.160%); 8582/172728 objects misplaced (4.969%); 6.5 MiB/s, 1 keys/s, 101 objects/s recovering 2023-10-09T04:40:22.003 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:22.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:22.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:21 smithi153 bash[59565]: cluster 2023-10-09T04:40:20.495781+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 31368/172728 objects degraded (18.160%); 8582/172728 objects misplaced (4.969%); 6.5 MiB/s, 1 keys/s, 101 objects/s recovering 2023-10-09T04:40:22.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:22.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:22.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:22.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:22.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:22.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:22.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:22.650 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:22.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:22.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:22.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:22.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:22.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:22.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:23.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:23.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:23.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:23.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:23.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:23.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:23.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:23.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:23.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:23.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:23.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:23.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:23.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:23.850 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:23 smithi082 bash[79733]: cluster 2023-10-09T04:40:22.496313+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 31368/169212 objects degraded (18.538%); 8582/169212 objects misplaced (5.072%); 2.7 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-09T04:40:23.851 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:23 smithi082 bash[77063]: cluster 2023-10-09T04:40:22.496313+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 31368/169212 objects degraded (18.538%); 8582/169212 objects misplaced (5.072%); 2.7 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-09T04:40:23.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:23.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:23.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:23.986 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:23 smithi153 bash[59565]: cluster 2023-10-09T04:40:22.496313+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 31368/169212 objects degraded (18.538%); 8582/169212 objects misplaced (5.072%); 2.7 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-09T04:40:23.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:24.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:24.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:24.195 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:24.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:24.274 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:24.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:24.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:24.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:24.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:24.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:24.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:24.715 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:24 smithi082 bash[77063]: cluster 2023-10-09T04:40:24.692712+0000 mon.a (mon.0) 1158 : cluster 3 Health check update: Degraded data redundancy: 31368/169212 objects degraded (18.538%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:24.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:24.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:24.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:24.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:24 smithi153 bash[59565]: cluster 2023-10-09T04:40:24.692712+0000 mon.a (mon.0) 1158 : cluster 3 Health check update: Degraded data redundancy: 31368/169212 objects degraded (18.538%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:24.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:24.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:24.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:24 smithi082 bash[79733]: cluster 2023-10-09T04:40:24.692712+0000 mon.a (mon.0) 1158 : cluster 3 Health check update: Degraded data redundancy: 31368/169212 objects degraded (18.538%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:25.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:25.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:25.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:25.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:25.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:25.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:25.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:25.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.584 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:25.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:25.709 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:25.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:25.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:25.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:25.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:25 smithi153 bash[59565]: cluster 2023-10-09T04:40:24.496993+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v800: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31130/162168 objects degraded (19.196%); 7882/162168 objects misplaced (4.860%); 5.1 MiB/s, 0 keys/s, 79 objects/s recovering 2023-10-09T04:40:25.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:25 smithi082 bash[77063]: cluster 2023-10-09T04:40:24.496993+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v800: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31130/162168 objects degraded (19.196%); 7882/162168 objects misplaced (4.860%); 5.1 MiB/s, 0 keys/s, 79 objects/s recovering 2023-10-09T04:40:25.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:25 smithi082 bash[79733]: cluster 2023-10-09T04:40:24.496993+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v800: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31130/162168 objects degraded (19.196%); 7882/162168 objects misplaced (4.860%); 5.1 MiB/s, 0 keys/s, 79 objects/s recovering 2023-10-09T04:40:25.971 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:25.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:26.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:26.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:26.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:26.227 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:26.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:26.360 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:26.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:26.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:26.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:26.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:26.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:26.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:26.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:26.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:26.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:26.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:26.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:27.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:27.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:27.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:27.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:27.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:27.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:27.389 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:27.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:27.518 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:27.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:27.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:27.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:27.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:27.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:27.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:27.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:27.912 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:27 smithi082 bash[79733]: cluster 2023-10-09T04:40:26.497636+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v801: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 31130/161571 objects degraded (19.267%); 7882/161571 objects misplaced (4.878%); 4.5 MiB/s, 0 keys/s, 68 objects/s recovering 2023-10-09T04:40:27.913 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:27 smithi082 bash[77063]: cluster 2023-10-09T04:40:26.497636+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v801: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 31130/161571 objects degraded (19.267%); 7882/161571 objects misplaced (4.878%); 4.5 MiB/s, 0 keys/s, 68 objects/s recovering 2023-10-09T04:40:27.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:27.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:28.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:28.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:27 smithi153 bash[59565]: cluster 2023-10-09T04:40:26.497636+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v801: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 31130/161571 objects degraded (19.267%); 7882/161571 objects misplaced (4.878%); 4.5 MiB/s, 0 keys/s, 68 objects/s recovering 2023-10-09T04:40:28.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:28.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:28.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:28.347 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:28.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:28.485 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:28.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:28.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:28.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:28.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:28.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:28.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:28.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:28.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:29.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:29.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:29.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:29.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:29.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:29.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:29.400 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:29.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:29.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:29.528 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:29.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:29.658 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:29.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:29.793 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:29.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:29.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:29.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[77063]: cluster 2023-10-09T04:40:28.498412+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v802: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 30906/152835 objects degraded (20.222%); 7147/152835 objects misplaced (4.676%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:40:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[77063]: audit 2023-10-09T04:40:29.278788+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:29.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[77063]: cluster 2023-10-09T04:40:29.693763+0000 mon.a (mon.0) 1160 : cluster 3 Health check update: Degraded data redundancy: 30906/152835 objects degraded (20.222%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[79733]: cluster 2023-10-09T04:40:28.498412+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v802: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 30906/152835 objects degraded (20.222%); 7147/152835 objects misplaced (4.676%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:40:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[79733]: audit 2023-10-09T04:40:29.278788+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[79733]: cluster 2023-10-09T04:40:29.693763+0000 mon.a (mon.0) 1160 : cluster 3 Health check update: Degraded data redundancy: 30906/152835 objects degraded (20.222%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:29] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-09T04:40:30.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:30.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:29 smithi153 bash[59565]: cluster 2023-10-09T04:40:28.498412+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v802: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 30906/152835 objects degraded (20.222%); 7147/152835 objects misplaced (4.676%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:40:30.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:29 smithi153 bash[59565]: audit 2023-10-09T04:40:29.278788+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:30.039 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:29 smithi153 bash[59565]: cluster 2023-10-09T04:40:29.693763+0000 mon.a (mon.0) 1160 : cluster 3 Health check update: Degraded data redundancy: 30906/152835 objects degraded (20.222%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:30.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:30.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:30.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:30.183 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:30.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:30.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:30.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:30.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:30.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:30.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:30.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:30.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:30.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:30.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:30.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:30.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:30.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:31.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:31.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:31.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:31.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:31.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:31.328 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:31.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:31.466 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:31.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:31.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:31.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:31.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:31.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:31.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:31.883 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:31.892 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:31 smithi153 bash[59565]: cluster 2023-10-09T04:40:30.499101+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v803: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 30906/150882 objects degraded (20.484%); 7147/150882 objects misplaced (4.737%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:31.912 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:31.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:31 smithi082 bash[77063]: cluster 2023-10-09T04:40:30.499101+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v803: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 30906/150882 objects degraded (20.484%); 7147/150882 objects misplaced (4.737%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:31 smithi082 bash[79733]: cluster 2023-10-09T04:40:30.499101+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v803: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 30906/150882 objects degraded (20.484%); 7147/150882 objects misplaced (4.737%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:32.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:32.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:32.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:32.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:32.274 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:32.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:32.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:32.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:32.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:32.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:32.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:32.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:32.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:32.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:32.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:33.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:33.108 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:33.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:33.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:33.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:33.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:33.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:33.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:33.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:33.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:33.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:33.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:33.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:33 smithi153 bash[59565]: cluster 2023-10-09T04:40:32.499712+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v804: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 30906/147543 objects degraded (20.947%); 7147/147543 objects misplaced (4.844%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:33.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:33.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:33.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:33 smithi082 bash[77063]: cluster 2023-10-09T04:40:32.499712+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v804: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 30906/147543 objects degraded (20.947%); 7147/147543 objects misplaced (4.844%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:33.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:33 smithi082 bash[79733]: cluster 2023-10-09T04:40:32.499712+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v804: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 30906/147543 objects degraded (20.947%); 7147/147543 objects misplaced (4.844%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:40:34.014 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:34.041 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:34.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:34.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:34.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:34.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:34.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:34.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:34.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:34.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:34.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:34.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:34.708 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:34.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:34.823 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:34 smithi082 bash[77063]: cluster 2023-10-09T04:40:34.694866+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 30906/147543 objects degraded (20.947%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:34.823 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:34 smithi082 bash[79733]: cluster 2023-10-09T04:40:34.694866+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 30906/147543 objects degraded (20.947%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:34.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:34.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:34.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:34.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:35.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:35.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:34 smithi153 bash[59565]: cluster 2023-10-09T04:40:34.694866+0000 mon.a (mon.0) 1161 : cluster 3 Health check update: Degraded data redundancy: 30906/147543 objects degraded (20.947%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:35.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:35.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:35.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:35.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:35.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:35.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:35.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:35.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:35.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:35.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:35.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:35.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:35.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:35.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:35.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:35.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:36.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:36.016 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:35 smithi153 bash[59565]: cluster 2023-10-09T04:40:34.500519+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v805: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 30693/140925 objects degraded (21.780%); 6407/140925 objects misplaced (4.546%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:40:36.032 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:36.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:36.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:36.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:35 smithi082 bash[77063]: cluster 2023-10-09T04:40:34.500519+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v805: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 30693/140925 objects degraded (21.780%); 6407/140925 objects misplaced (4.546%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:40:36.175 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:35 smithi082 bash[79733]: cluster 2023-10-09T04:40:34.500519+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v805: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 30693/140925 objects degraded (21.780%); 6407/140925 objects misplaced (4.546%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:40:36.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:36.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:36.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:36.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:36.525 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:36.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:36.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:36.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:36.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:36.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:36.819 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:36.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:36.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:36.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:36.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:37.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:37.080 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:37.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:37.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:37.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:37.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:37.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:37.591 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:37.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:37.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:37.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.864 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:37 smithi153 bash[59565]: cluster 2023-10-09T04:40:36.501208+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v806: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 30693/140295 objects degraded (21.877%); 6407/140295 objects misplaced (4.567%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:40:37.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:37.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:37.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:38.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:38.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.134 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:37 smithi082 bash[77063]: cluster 2023-10-09T04:40:36.501208+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v806: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 30693/140295 objects degraded (21.877%); 6407/140295 objects misplaced (4.567%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:40:38.134 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:37 smithi082 bash[79733]: cluster 2023-10-09T04:40:36.501208+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v806: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 30693/140295 objects degraded (21.877%); 6407/140295 objects misplaced (4.567%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:40:38.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:38.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:38.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:38.406 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:38.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:38.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:38.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:38.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:38.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:38.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:38.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:38.962 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:38.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:38.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:39.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:39.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:39.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:39.223 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:39.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:39.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:39.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:39.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:39.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:39.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:39.627 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:39.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:39] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-09T04:40:39.727 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:40:39.727 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:40:39.727 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.727 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:163313 7.11 e193/186 7:896a4c9a:::benchmark_data_smithi082_155217_object60653:head v 193'9037, mlcod=179'4605)", 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:39.719603+0000", 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0056383609999999997, 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056726820000000001, 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:40:39.728 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719603+0000", 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719603+0000", 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.729 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719602+0000", 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719612+0000", 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.9410000000000002e-06 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.730 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719627+0000", 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.466e-05 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.719634+0000", 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9840000000000004e-06 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:40:39.731 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724945+0000", 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053110070000000004 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724971+0000", 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5803999999999999e-05 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:163307 7.1 e193/187)", 2023-10-09T04:40:39.732 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:39.724929+0000", 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00031260000000000001, 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00041589600000000002, 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724929+0000", 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.733 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724929+0000", 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724928+0000", 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:40:39.734 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724937+0000", 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3920000000000003e-06 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724946+0000", 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2650000000000002e-06 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.735 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724951+0000", 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3580000000000002e-06 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.736 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:163320 7.17 e193/186 7:eb2b4076:::benchmark_data_smithi082_155217_object60660:head v 193'9237, mlcod=193'9235)", 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:39.724956+0000", 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00028534399999999998, 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000448104, 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:40:39.737 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724956+0000", 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.738 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724956+0000", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724956+0000", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967296 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724959+0000", 2023-10-09T04:40:39.739 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4400000000000001e-06 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724963+0000", 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3069999999999998e-06 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.724965+0000", 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5170000000000002e-06 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.740 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.725116+0000", 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015092999999999999 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:39.725136+0000", 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0231e-05 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:40:39.741 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 3 2023-10-09T04:40:39.742 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:40:39.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:39.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:39.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:39.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:39.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:39.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:39 smithi082 bash[77063]: cluster 2023-10-09T04:40:38.501981+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v807: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 30462/131886 objects degraded (23.097%); 5710/131886 objects misplaced (4.329%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:40:39.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:39 smithi082 bash[77063]: cluster 2023-10-09T04:40:39.696221+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 30462/131886 objects degraded (23.097%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:39 smithi082 bash[79733]: cluster 2023-10-09T04:40:38.501981+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v807: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 30462/131886 objects degraded (23.097%); 5710/131886 objects misplaced (4.329%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:40:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:39 smithi082 bash[79733]: cluster 2023-10-09T04:40:39.696221+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 30462/131886 objects degraded (23.097%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:40.007 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:40.016 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:39 smithi153 bash[59565]: cluster 2023-10-09T04:40:38.501981+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v807: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 30462/131886 objects degraded (23.097%); 5710/131886 objects misplaced (4.329%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:40:40.016 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:39 smithi153 bash[59565]: cluster 2023-10-09T04:40:39.696221+0000 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 30462/131886 objects degraded (23.097%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:40.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:40.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:40.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:40.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:40.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:40.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:40.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:40.443 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:40.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:40.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:40.586 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:40.689 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:40.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:40.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:40.825 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:40.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:40.954 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:40.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:40.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:41.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:41.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:41.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:41.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:41.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:41.378 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:41.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:41.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:41.598 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:40:41.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:41.635 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:40:41.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:41.678 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:40:41.721 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:40:41.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:41.762 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:40:41.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:41.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:41.798 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:40:41.831 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:40:41.864 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:40:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.872 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.872 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.872 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:40:41.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:41.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:41 smithi153 bash[59565]: cluster 2023-10-09T04:40:40.502687+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 30462/129960 objects degraded (23.440%); 5710/129960 objects misplaced (4.394%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:41.884 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:41 smithi153 bash[59565]: audit 2023-10-09T04:40:41.495145+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.82:0/3968645913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:41.898 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:41.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:42.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:42.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:42.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:42.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:42.162 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:42.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:41 smithi082 bash[77063]: cluster 2023-10-09T04:40:40.502687+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 30462/129960 objects degraded (23.440%); 5710/129960 objects misplaced (4.394%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:42.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:41 smithi082 bash[77063]: audit 2023-10-09T04:40:41.495145+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.82:0/3968645913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:41 smithi082 bash[79733]: cluster 2023-10-09T04:40:40.502687+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v808: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 30462/129960 objects degraded (23.440%); 5710/129960 objects misplaced (4.394%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:41 smithi082 bash[79733]: audit 2023-10-09T04:40:41.495145+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.82:0/3968645913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:42.266 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:42.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:42.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:42.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:42.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:42.532 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:42.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:42.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:42.701 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:42.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.837 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:42.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:42.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:42.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:42.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:43.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:43.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:43.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:43.227 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:43.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:43.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:43.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:43.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:43.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:43.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:43.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:43.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:43.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:43.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:43.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:43.887 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:43 smithi082 bash[77063]: cluster 2023-10-09T04:40:42.503401+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 30462/126849 objects degraded (24.014%); 5710/126849 objects misplaced (4.501%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:43.887 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:43 smithi082 bash[79733]: cluster 2023-10-09T04:40:42.503401+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 30462/126849 objects degraded (24.014%); 5710/126849 objects misplaced (4.501%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:43.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:40:43.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:43.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:43.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:44.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:44.029 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:43 smithi153 bash[59565]: cluster 2023-10-09T04:40:42.503401+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v809: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 30462/126849 objects degraded (24.014%); 5710/126849 objects misplaced (4.501%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:44.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:44.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:44.280 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:44.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:44.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:44.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:44.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:44.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:44.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:44.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:44.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:44.954 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:44 smithi082 bash[79733]: audit 2023-10-09T04:40:44.279001+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:44.954 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:44 smithi082 bash[79733]: cluster 2023-10-09T04:40:44.697319+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 30462/126849 objects degraded (24.014%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:44.954 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:44 smithi082 bash[77063]: audit 2023-10-09T04:40:44.279001+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:44.954 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:44 smithi082 bash[77063]: cluster 2023-10-09T04:40:44.697319+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 30462/126849 objects degraded (24.014%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:44.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:44.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:45.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:45.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:44 smithi153 bash[59565]: audit 2023-10-09T04:40:44.279001+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:45.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:44 smithi153 bash[59565]: cluster 2023-10-09T04:40:44.697319+0000 mon.a (mon.0) 1164 : cluster 3 Health check update: Degraded data redundancy: 30462/126849 objects degraded (24.014%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:45.113 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:45.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:45.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:45.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:45.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:45.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:45.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:45.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:45.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:45.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:45.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:45.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:45 smithi153 bash[59565]: cluster 2023-10-09T04:40:44.504183+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 30247/120270 objects degraded (25.149%); 4992/120270 objects misplaced (4.151%); 7.8 MiB/s, 128 objects/s recovering 2023-10-09T04:40:45.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:45.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:46.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:46.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:46.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:46.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:45 smithi082 bash[77063]: cluster 2023-10-09T04:40:44.504183+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 30247/120270 objects degraded (25.149%); 4992/120270 objects misplaced (4.151%); 7.8 MiB/s, 128 objects/s recovering 2023-10-09T04:40:46.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:45 smithi082 bash[79733]: cluster 2023-10-09T04:40:44.504183+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v810: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 30247/120270 objects degraded (25.149%); 4992/120270 objects misplaced (4.151%); 7.8 MiB/s, 128 objects/s recovering 2023-10-09T04:40:46.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:46.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:46.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:46.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:46.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:46.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:46.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:46.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[77063]: debug 2023-10-09T04:40:46.230+0000 7f3fc98c6700 -1 mon.a@0(leader).osd e193 definitely_dead 0 2023-10-09T04:40:46.815 INFO:teuthology.orchestra.run.smithi082.stderr:marked in osd.7. 2023-10-09T04:40:46.815 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:40:46.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:46.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:46 smithi153 bash[59565]: audit 2023-10-09T04:40:46.014724+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:46.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:46 smithi153 bash[59565]: audit 2023-10-09T04:40:46.231208+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.82:0/581831484' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:46.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:46 smithi153 bash[59565]: audit 2023-10-09T04:40:46.233731+0000 mon.a (mon.0) 1166 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:46.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:46.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:46.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:47.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:47.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[79733]: audit 2023-10-09T04:40:46.014724+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:47.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[79733]: audit 2023-10-09T04:40:46.231208+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.82:0/581831484' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:47.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[79733]: audit 2023-10-09T04:40:46.233731+0000 mon.a (mon.0) 1166 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:47.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[77063]: audit 2023-10-09T04:40:46.014724+0000 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:47.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[77063]: audit 2023-10-09T04:40:46.231208+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.82:0/581831484' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:47.107 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:46 smithi082 bash[77063]: audit 2023-10-09T04:40:46.233731+0000 mon.a (mon.0) 1166 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T04:40:47.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:47.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:47.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:47.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:47.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:40:47.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:47.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:47.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:47.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:47.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:47.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:47.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: cluster 2023-10-09T04:40:46.504813+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 30247/119652 objects degraded (25.279%); 4992/119652 objects misplaced (4.172%); 5.3 MiB/s, 87 objects/s recovering 2023-10-09T04:40:47.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: cluster 2023-10-09T04:40:46.778446+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:47.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: cluster 2023-10-09T04:40:46.794666+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:47.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: cluster 2023-10-09T04:40:46.799276+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:47.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: audit 2023-10-09T04:40:46.801349+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:47.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: audit 2023-10-09T04:40:46.801693+0000 mon.a (mon.0) 1169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T04:40:47.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:47 smithi153 bash[59565]: cluster 2023-10-09T04:40:46.804360+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:40:47.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:47.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:48.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: cluster 2023-10-09T04:40:46.504813+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 30247/119652 objects degraded (25.279%); 4992/119652 objects misplaced (4.172%); 5.3 MiB/s, 87 objects/s recovering 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: cluster 2023-10-09T04:40:46.778446+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: cluster 2023-10-09T04:40:46.794666+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: cluster 2023-10-09T04:40:46.799276+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: audit 2023-10-09T04:40:46.801349+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: audit 2023-10-09T04:40:46.801693+0000 mon.a (mon.0) 1169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T04:40:48.029 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[79733]: cluster 2023-10-09T04:40:46.804360+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: cluster 2023-10-09T04:40:46.504813+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v811: 99 pgs: 1 active+recovering, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 30247/119652 objects degraded (25.279%); 4992/119652 objects misplaced (4.172%); 5.3 MiB/s, 87 objects/s recovering 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: cluster 2023-10-09T04:40:46.778446+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: cluster 2023-10-09T04:40:46.794666+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: cluster 2023-10-09T04:40:46.799276+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: audit 2023-10-09T04:40:46.801349+0000 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.82:0/3695834445' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: audit 2023-10-09T04:40:46.801693+0000 mon.a (mon.0) 1169 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T04:40:48.030 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:47 smithi082 bash[77063]: cluster 2023-10-09T04:40:46.804360+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:40:48.049 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:48.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:48.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:48.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:48.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:48.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:48.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:48.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:48.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:48.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:48.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:48.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:48.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:48.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.795423+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.797164+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.798278+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.798303+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.800241+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.800593+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.800894+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.801820+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.803519+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:40:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:47.805433+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:48.927 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:48 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.798993+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:40:48.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:49.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:49.106 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.795423+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.797164+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.798278+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.798303+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.800241+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.800593+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.800894+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.801820+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.803519+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:40:49.115 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:47.805433+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.798993+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.795423+0000 osd.4 (osd.4) 138 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.797164+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.798278+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.798303+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.800241+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.800593+0000 osd.4 (osd.4) 139 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.800894+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.801820+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.116 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.803519+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:40:49.117 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:47.805433+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T04:40:49.117 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:48 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.798993+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:40:49.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:49.238 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:49.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:49.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:49.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:49.508 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T04:40:49.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:49.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:49.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:49.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:49.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:49.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:49.822 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:49] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.33.4" 2023-10-09T04:40:49.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:49.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.505546+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v814: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 8955/112479 objects degraded (7.961%); 572/112479 objects misplaced (0.509%); 6.5 MiB/s, 150 objects/s recovering 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.796516+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.797088+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.799272+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.802810+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:48.806508+0000 mon.a (mon.0) 1173 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:40:49.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:49 smithi153 bash[59565]: cluster 2023-10-09T04:40:49.698404+0000 mon.a (mon.0) 1174 : cluster 3 Health check update: Degraded data redundancy: 8955/112479 objects degraded (7.961%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:49.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:49.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:50.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:50.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:50.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:50.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.505546+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v814: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 8955/112479 objects degraded (7.961%); 572/112479 objects misplaced (0.509%); 6.5 MiB/s, 150 objects/s recovering 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.796516+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.797088+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.799272+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.802810+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:48.806508+0000 mon.a (mon.0) 1173 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[77063]: cluster 2023-10-09T04:40:49.698404+0000 mon.a (mon.0) 1174 : cluster 3 Health check update: Degraded data redundancy: 8955/112479 objects degraded (7.961%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.505546+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v814: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 83 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 8955/112479 objects degraded (7.961%); 572/112479 objects misplaced (0.509%); 6.5 MiB/s, 150 objects/s recovering 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.796516+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.797088+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.799272+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.802810+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T04:40:50.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:48.806508+0000 mon.a (mon.0) 1173 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:40:50.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:49 smithi082 bash[79733]: cluster 2023-10-09T04:40:49.698404+0000 mon.a (mon.0) 1174 : cluster 3 Health check update: Degraded data redundancy: 8955/112479 objects degraded (7.961%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:50.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:50.362 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:50.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:50.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:50.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:50.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:50.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:50.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:50.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:50.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:50.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:50.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:50.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:51.022 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:51.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:51.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:51.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:51.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:51.324 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:51.432 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:51.457 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:51.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:40:51.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:51.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:51.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:51.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:51.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:51.828 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:51 smithi082 bash[77063]: cluster 2023-10-09T04:40:50.506580+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v816: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8971/110982 objects degraded (8.083%); 572/110982 objects misplaced (0.515%); 3.4 MiB/s, 110 objects/s recovering 2023-10-09T04:40:51.828 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:51 smithi082 bash[79733]: cluster 2023-10-09T04:40:50.506580+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v816: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8971/110982 objects degraded (8.083%); 572/110982 objects misplaced (0.515%); 3.4 MiB/s, 110 objects/s recovering 2023-10-09T04:40:51.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:51.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:51.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:51.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.012 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.013 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:52.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:52.124 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:51 smithi153 bash[59565]: cluster 2023-10-09T04:40:50.506580+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v816: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 81 active+clean; 2.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 8971/110982 objects degraded (8.083%); 572/110982 objects misplaced (0.515%); 3.4 MiB/s, 110 objects/s recovering 2023-10-09T04:40:52.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:52.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:52.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:52.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:52.428 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:52.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:52.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:52.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:52.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:52.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:52.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:52.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:52.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:52.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:52.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:53.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:53.209 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:53.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:53.367 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:53.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:53.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:53.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:53.759 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:53.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:53.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:53.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:53.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:53.999 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:53 smithi082 bash[77063]: cluster 2023-10-09T04:40:52.507324+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 84 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 11220/107892 objects degraded (10.399%); 572/107892 objects misplaced (0.530%); 3.4 MiB/s, 171 objects/s recovering 2023-10-09T04:40:54.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:53 smithi082 bash[79733]: cluster 2023-10-09T04:40:52.507324+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 84 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 11220/107892 objects degraded (10.399%); 572/107892 objects misplaced (0.530%); 3.4 MiB/s, 171 objects/s recovering 2023-10-09T04:40:54.029 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:54.051 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:54.132 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:54.141 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:53 smithi153 bash[59565]: cluster 2023-10-09T04:40:52.507324+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v817: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 84 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 11220/107892 objects degraded (10.399%); 572/107892 objects misplaced (0.530%); 3.4 MiB/s, 171 objects/s recovering 2023-10-09T04:40:54.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:54.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:54.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:54.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:54.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:54.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:54.509 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:40:54.509 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:40:54.509 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:40:54.509 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.418784 2023-10-09T04:40:54.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0.41878403184908564 2023-10-09T04:40:54.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:54.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:54.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:54.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:54.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:54.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:54.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:54.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:54.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:54.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:54.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:55.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:55.088 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:54 smithi153 bash[59565]: cluster 2023-10-09T04:40:54.699560+0000 mon.a (mon.0) 1175 : cluster 3 Health check update: Degraded data redundancy: 11220/107892 objects degraded (10.399%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:55.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:55.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:55.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:55.270 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:54 smithi082 bash[77063]: cluster 2023-10-09T04:40:54.699560+0000 mon.a (mon.0) 1175 : cluster 3 Health check update: Degraded data redundancy: 11220/107892 objects degraded (10.399%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:55.270 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:54 smithi082 bash[79733]: cluster 2023-10-09T04:40:54.699560+0000 mon.a (mon.0) 1175 : cluster 3 Health check update: Degraded data redundancy: 11220/107892 objects degraded (10.399%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:55.335 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:55.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:55.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:55.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:55.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:55.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:55.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:55.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:55.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:55.901 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:55 smithi082 bash[77063]: cluster 2023-10-09T04:40:54.508079+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 12498/100203 objects degraded (12.473%); 3.6 MiB/s, 149 objects/s recovering 2023-10-09T04:40:55.902 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:55 smithi082 bash[77063]: cluster 2023-10-09T04:40:54.828440+0000 mon.a (mon.0) 1176 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:40:55.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:55 smithi082 bash[79733]: cluster 2023-10-09T04:40:54.508079+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 12498/100203 objects degraded (12.473%); 3.6 MiB/s, 149 objects/s recovering 2023-10-09T04:40:55.902 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:55 smithi082 bash[79733]: cluster 2023-10-09T04:40:54.828440+0000 mon.a (mon.0) 1176 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:40:55.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:55.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:56.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:56.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:56.088 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:56.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:55 smithi153 bash[59565]: cluster 2023-10-09T04:40:54.508079+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v818: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 12498/100203 objects degraded (12.473%); 3.6 MiB/s, 149 objects/s recovering 2023-10-09T04:40:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:55 smithi153 bash[59565]: cluster 2023-10-09T04:40:54.828440+0000 mon.a (mon.0) 1176 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:40:56.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:56.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:56.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:56.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:56.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:56.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:56.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:56.669 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:56.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:56.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:56.819 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:56.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:56.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:56.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:57.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:57.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:57.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:56 smithi153 bash[59565]: cluster 2023-10-09T04:40:55.844431+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:56 smithi153 bash[59565]: cluster 2023-10-09T04:40:55.847015+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:56 smithi153 bash[59565]: cluster 2023-10-09T04:40:55.847653+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:56 smithi153 bash[59565]: cluster 2023-10-09T04:40:55.848029+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:40:57.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:56 smithi153 bash[59565]: cluster 2023-10-09T04:40:55.854050+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:57.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[77063]: cluster 2023-10-09T04:40:55.844431+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[77063]: cluster 2023-10-09T04:40:55.847015+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[77063]: cluster 2023-10-09T04:40:55.847653+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[77063]: cluster 2023-10-09T04:40:55.848029+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:40:57.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[77063]: cluster 2023-10-09T04:40:55.854050+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.200 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[79733]: cluster 2023-10-09T04:40:55.844431+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.200 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[79733]: cluster 2023-10-09T04:40:55.847015+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.200 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[79733]: cluster 2023-10-09T04:40:55.847653+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.201 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[79733]: cluster 2023-10-09T04:40:55.848029+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:40:57.201 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:56 smithi082 bash[79733]: cluster 2023-10-09T04:40:55.854050+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T04:40:57.229 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:57.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:57.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:57.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:57.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:57.612 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:57.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:57.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:57.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:57.881 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:57.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:57.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:57.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.508578+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12498/99393 objects degraded (12.574%); 960 KiB/s, 61 objects/s recovering 2023-10-09T04:40:57.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.851138+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.853340+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.853341+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.854225+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.508578+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12498/99393 objects degraded (12.574%); 960 KiB/s, 61 objects/s recovering 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.851138+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.853340+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.853341+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:57 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.854225+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:40:58.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:58.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:57 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.508578+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v820: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 84 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12498/99393 objects degraded (12.574%); 960 KiB/s, 61 objects/s recovering 2023-10-09T04:40:58.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:57 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.851138+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:57 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.853340+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:57 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.853341+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.027 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:57 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.854225+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:40:58.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:58.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:58.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:58.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:58.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:58.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:58.464 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:58.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:58.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:58.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:58.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:58.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:58.870 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:58.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:58.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:58.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:58 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.862379+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:58 smithi082 bash[77063]: cluster 2023-10-09T04:40:56.863827+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:58 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.862379+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:58.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:58 smithi082 bash[79733]: cluster 2023-10-09T04:40:56.863827+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:59.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:59.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:59.152 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:59.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:58 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.862379+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:59.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:58 smithi153 bash[59565]: cluster 2023-10-09T04:40:56.863827+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T04:40:59.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:59.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:59.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:59.417 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:59.527 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.556 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:59.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:59.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:40:59] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.33.4" 2023-10-09T04:40:59.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:59.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:59 smithi153 bash[59565]: cluster 2023-10-09T04:40:58.509372+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11557/90783 objects degraded (12.730%); 2.4 MiB/s, 170 objects/s recovering 2023-10-09T04:40:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:59 smithi153 bash[59565]: audit 2023-10-09T04:40:59.279573+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:59.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:40:59 smithi153 bash[59565]: cluster 2023-10-09T04:40:59.700842+0000 mon.a (mon.0) 1180 : cluster 3 Health check update: Degraded data redundancy: 11557/90783 objects degraded (12.730%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:59.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:59.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:40:59.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[77063]: cluster 2023-10-09T04:40:58.509372+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11557/90783 objects degraded (12.730%); 2.4 MiB/s, 170 objects/s recovering 2023-10-09T04:40:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[77063]: audit 2023-10-09T04:40:59.279573+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:59.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[77063]: cluster 2023-10-09T04:40:59.700842+0000 mon.a (mon.0) 1180 : cluster 3 Health check update: Degraded data redundancy: 11557/90783 objects degraded (12.730%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[79733]: cluster 2023-10-09T04:40:58.509372+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v822: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11557/90783 objects degraded (12.730%); 2.4 MiB/s, 170 objects/s recovering 2023-10-09T04:40:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[79733]: audit 2023-10-09T04:40:59.279573+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:59.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:40:59 smithi082 bash[79733]: cluster 2023-10-09T04:40:59.700842+0000 mon.a (mon.0) 1180 : cluster 3 Health check update: Degraded data redundancy: 11557/90783 objects degraded (12.730%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:00.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:00.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:00.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:00.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:00.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:00.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:00.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:00.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:00.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:00.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:00.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:00.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:00.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:01.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:01.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:01.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:01.266 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:01.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:01.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:01.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:01.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:01.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:01.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:01.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:01.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:01.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:01.869 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:01 smithi153 bash[59565]: cluster 2023-10-09T04:41:00.510107+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11557/88881 objects degraded (13.003%); 2.4 MiB/s, 124 objects/s recovering 2023-10-09T04:41:01.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:01.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:01.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:02.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:02.109 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:02.213 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:01 smithi082 bash[77063]: cluster 2023-10-09T04:41:00.510107+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11557/88881 objects degraded (13.003%); 2.4 MiB/s, 124 objects/s recovering 2023-10-09T04:41:02.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:02 smithi082 bash[79733]: cluster 2023-10-09T04:41:00.510107+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v823: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 11557/88881 objects degraded (13.003%); 2.4 MiB/s, 124 objects/s recovering 2023-10-09T04:41:02.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:02.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:02.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:02.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:02.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.632 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:02.736 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:02.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:02.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:02.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:02.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:03.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:03.029 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:03.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:03.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:03.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:03.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:03.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:03.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:03.539 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:03.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:03.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:03.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:03.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:03.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:03.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:03.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:03.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:03 smithi082 bash[77063]: cluster 2023-10-09T04:41:02.513380+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11557/85293 objects degraded (13.550%); 1.4 MiB/s, 109 objects/s recovering 2023-10-09T04:41:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:03 smithi082 bash[79733]: cluster 2023-10-09T04:41:02.513380+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11557/85293 objects degraded (13.550%); 1.4 MiB/s, 109 objects/s recovering 2023-10-09T04:41:03.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:03.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:04.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:04.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:03 smithi153 bash[59565]: cluster 2023-10-09T04:41:02.513380+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v824: 99 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 85 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11557/85293 objects degraded (13.550%); 1.4 MiB/s, 109 objects/s recovering 2023-10-09T04:41:04.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:04.218 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:04.247 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:04.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:04.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:04.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:04.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:04.614 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:04.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:04.748 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:04.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:04.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:04.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:04.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:04.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:04.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:05.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:05.039 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:05.142 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:05.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[79733]: cluster 2023-10-09T04:41:03.882547+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[79733]: cluster 2023-10-09T04:41:03.883061+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[79733]: cluster 2023-10-09T04:41:03.887218+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:41:05.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[79733]: cluster 2023-10-09T04:41:03.896711+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.704035+0000 mon.a (mon.0) 1182 : cluster 3 Health check update: Degraded data redundancy: 11557/85293 objects degraded (13.550%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:05.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[77063]: cluster 2023-10-09T04:41:03.882547+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[77063]: cluster 2023-10-09T04:41:03.883061+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[77063]: cluster 2023-10-09T04:41:03.887218+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:41:05.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[77063]: cluster 2023-10-09T04:41:03.896711+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:04 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.704035+0000 mon.a (mon.0) 1182 : cluster 3 Health check update: Degraded data redundancy: 11557/85293 objects degraded (13.550%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:04 smithi153 bash[59565]: cluster 2023-10-09T04:41:03.882547+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:04 smithi153 bash[59565]: cluster 2023-10-09T04:41:03.883061+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:04 smithi153 bash[59565]: cluster 2023-10-09T04:41:03.887218+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:41:05.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:04 smithi153 bash[59565]: cluster 2023-10-09T04:41:03.896711+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T04:41:05.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:04 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.704035+0000 mon.a (mon.0) 1182 : cluster 3 Health check update: Degraded data redundancy: 11557/85293 objects degraded (13.550%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:05.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:05.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:05.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:05.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:05.413 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:05.452 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:05.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:05.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:05.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:05.715 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:05.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:05.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:05.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:05.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:06.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:06.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:06.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:06.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:06.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.513872+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v826: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 85 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 10001/78069 objects degraded (12.810%); 462/78069 objects misplaced (0.592%); 3.9 MiB/s, 204 objects/s recovering 2023-10-09T04:41:06.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.892299+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.892926+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.893853+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.893958+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.895973+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.898110+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.901571+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.904262+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.905905+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:05 smithi153 bash[59565]: cluster 2023-10-09T04:41:04.915761+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.513872+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v826: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 85 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 10001/78069 objects degraded (12.810%); 462/78069 objects misplaced (0.592%); 3.9 MiB/s, 204 objects/s recovering 2023-10-09T04:41:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.892299+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.892926+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.893853+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.893958+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.895973+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.898110+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.901571+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.904262+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.905905+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[77063]: cluster 2023-10-09T04:41:04.915761+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.513872+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v826: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 85 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 10001/78069 objects degraded (12.810%); 462/78069 objects misplaced (0.592%); 3.9 MiB/s, 204 objects/s recovering 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.892299+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.892926+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.893853+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.893958+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.895973+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.898110+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.901571+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.904262+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.905905+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:05 smithi082 bash[79733]: cluster 2023-10-09T04:41:04.915761+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T04:41:06.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:06.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:06.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:06.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:06.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:06.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:06.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:06.678 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:06.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:06.784 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:06.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:06.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:06.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:06.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:07.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:07.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:07.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:07.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:07.298 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:07.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:07.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:07.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:07.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:07.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:07.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:07.619 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:07.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:07.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:07.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:07.863 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:07.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:07.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:07.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:08.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:08.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:08.130 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:07 smithi082 bash[77063]: cluster 2023-10-09T04:41:06.514589+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v828: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 86 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 9994/77379 objects degraded (12.916%); 462/77379 objects misplaced (0.597%); 2.9 MiB/s, 96 objects/s recovering 2023-10-09T04:41:08.130 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:07 smithi082 bash[79733]: cluster 2023-10-09T04:41:06.514589+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v828: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 86 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 9994/77379 objects degraded (12.916%); 462/77379 objects misplaced (0.597%); 2.9 MiB/s, 96 objects/s recovering 2023-10-09T04:41:08.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:08.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:07 smithi153 bash[59565]: cluster 2023-10-09T04:41:06.514589+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v828: 99 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 86 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 9994/77379 objects degraded (12.916%); 462/77379 objects misplaced (0.597%); 2.9 MiB/s, 96 objects/s recovering 2023-10-09T04:41:08.246 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:08.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:08.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:08.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:08.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:08.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:08.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:08.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:08.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:08.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:08.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:08.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:08.912 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:08.912 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:08.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:08.930 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:08.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:09.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:09.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:09.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:09.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:09.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:09.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:09.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:09.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:09.456 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:09.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:09.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:09.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:09.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:09] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.33.4" 2023-10-09T04:41:09.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:09.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:09.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:09.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:09.963 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:09.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[77063]: cluster 2023-10-09T04:41:08.515394+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 9049/69105 objects degraded (13.095%); 75/69105 objects misplaced (0.109%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[77063]: cluster 2023-10-09T04:41:09.704926+0000 mon.a (mon.0) 1184 : cluster 3 Health check update: Degraded data redundancy: 9049/69105 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:09.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[77063]: audit 2023-10-09T04:41:09.869966+0000 mon.a (mon.0) 1185 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[79733]: cluster 2023-10-09T04:41:08.515394+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 9049/69105 objects degraded (13.095%); 75/69105 objects misplaced (0.109%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[79733]: cluster 2023-10-09T04:41:09.704926+0000 mon.a (mon.0) 1184 : cluster 3 Health check update: Degraded data redundancy: 9049/69105 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:09 smithi082 bash[79733]: audit 2023-10-09T04:41:09.869966+0000 mon.a (mon.0) 1185 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:09.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:10.046 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:41:10.090 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:41:10.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:10.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:09 smithi153 bash[59565]: cluster 2023-10-09T04:41:08.515394+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v829: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 9049/69105 objects degraded (13.095%); 75/69105 objects misplaced (0.109%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:10.110 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:09 smithi153 bash[59565]: cluster 2023-10-09T04:41:09.704926+0000 mon.a (mon.0) 1184 : cluster 3 Health check update: Degraded data redundancy: 9049/69105 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:10.111 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:09 smithi153 bash[59565]: audit 2023-10-09T04:41:09.869966+0000 mon.a (mon.0) 1185 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:10.128 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:41:10.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:10.163 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:41:10.205 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:41:10.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:10.249 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:41:10.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:10.298 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:41:10.340 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:41:10.342 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.342 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.342 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.342 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.343 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.343 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.343 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.343 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:41:10.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:10.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:10.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:10.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:10.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:10.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:10.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:10.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:10.803 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:10.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:10.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:10.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:11.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:11.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:11.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:10 smithi153 bash[59565]: audit 2023-10-09T04:41:09.989523+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.82:0/3877967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:11.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:10 smithi082 bash[77063]: audit 2023-10-09T04:41:09.989523+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.82:0/3877967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:11.176 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:10 smithi082 bash[79733]: audit 2023-10-09T04:41:09.989523+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.82:0/3877967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:11.193 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:11.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:11.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:11.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:11.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:11.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:11.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:11.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:11.719 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:11.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:11.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:11.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:11.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:11.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:12.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:12.105 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:11 smithi153 bash[59565]: cluster 2023-10-09T04:41:10.516072+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 9049/67341 objects degraded (13.438%); 75/67341 objects misplaced (0.111%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:12.126 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:12.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:12.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:12.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:11 smithi082 bash[77063]: cluster 2023-10-09T04:41:10.516072+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 9049/67341 objects degraded (13.438%); 75/67341 objects misplaced (0.111%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:11 smithi082 bash[79733]: cluster 2023-10-09T04:41:10.516072+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v830: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 9049/67341 objects degraded (13.438%); 75/67341 objects misplaced (0.111%); 7.5 MiB/s, 219 objects/s recovering 2023-10-09T04:41:12.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:12.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:12.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:12.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:12.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:12.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:12.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:12.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:12.736 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:12.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:12.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:12.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:12.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:12.943 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[77063]: cluster 2023-10-09T04:41:11.934561+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.943 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[77063]: cluster 2023-10-09T04:41:11.939416+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[77063]: cluster 2023-10-09T04:41:11.941814+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[77063]: cluster 2023-10-09T04:41:11.944378+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[77063]: cluster 2023-10-09T04:41:11.987776+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[79733]: cluster 2023-10-09T04:41:11.934561+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[79733]: cluster 2023-10-09T04:41:11.939416+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[79733]: cluster 2023-10-09T04:41:11.941814+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[79733]: cluster 2023-10-09T04:41:11.944378+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.944 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:12 smithi082 bash[79733]: cluster 2023-10-09T04:41:11.987776+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:12.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:13.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:12 smithi153 bash[59565]: cluster 2023-10-09T04:41:11.934561+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:13.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:12 smithi153 bash[59565]: cluster 2023-10-09T04:41:11.939416+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:41:13.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:12 smithi153 bash[59565]: cluster 2023-10-09T04:41:11.941814+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:13.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:12 smithi153 bash[59565]: cluster 2023-10-09T04:41:11.944378+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:13.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:12 smithi153 bash[59565]: cluster 2023-10-09T04:41:11.987776+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T04:41:13.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:13.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:13.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:13.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:13.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:13.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:13.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:13.433 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:13.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:13.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:13.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:13.678 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:13.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:13.817 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:13.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:13.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:13.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:13.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:14.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:41:14.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:14.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.516566+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v832: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 8298/64218 objects degraded (12.922%); 75/64218 objects misplaced (0.117%); 5.1 MiB/s, 139 objects/s recovering 2023-10-09T04:41:14.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.944820+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.944995+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:41:14.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.945038+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.947549+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.092 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:12.948167+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.092 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[77063]: cluster 2023-10-09T04:41:13.068584+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.516566+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v832: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 8298/64218 objects degraded (12.922%); 75/64218 objects misplaced (0.117%); 5.1 MiB/s, 139 objects/s recovering 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.944820+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.944995+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.945038+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.947549+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:12.948167+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.093 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:13 smithi082 bash[79733]: cluster 2023-10-09T04:41:13.068584+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:14.245 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:14.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:14.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:14.404 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.516566+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v832: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 8298/64218 objects degraded (12.922%); 75/64218 objects misplaced (0.117%); 5.1 MiB/s, 139 objects/s recovering 2023-10-09T04:41:14.404 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.944820+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.405 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.944995+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:41:14.405 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.945038+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.405 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.947549+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.405 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:12.948167+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.405 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:13 smithi153 bash[59565]: cluster 2023-10-09T04:41:13.068584+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T04:41:14.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:14.524 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:14.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:14.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:14.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:14.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:14.813 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:14.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:14.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:14.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:15.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:14 smithi153 bash[59565]: audit 2023-10-09T04:41:14.279819+0000 mon.a (mon.0) 1188 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:15.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:14 smithi153 bash[59565]: cluster 2023-10-09T04:41:14.705972+0000 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 8298/64218 objects degraded (12.922%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:15.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:15.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:14 smithi082 bash[77063]: audit 2023-10-09T04:41:14.279819+0000 mon.a (mon.0) 1188 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:15.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:14 smithi082 bash[77063]: cluster 2023-10-09T04:41:14.705972+0000 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 8298/64218 objects degraded (12.922%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:14 smithi082 bash[79733]: audit 2023-10-09T04:41:14.279819+0000 mon.a (mon.0) 1188 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:15.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:14 smithi082 bash[79733]: cluster 2023-10-09T04:41:14.705972+0000 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 8298/64218 objects degraded (12.922%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:15.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:15.385 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:15.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:15.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:15.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:15.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:15.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:15.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:15.991 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:41:16.038 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[77063]: cluster 2023-10-09T04:41:14.517308+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v834: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8106/58362 objects degraded (13.889%); 6.7 MiB/s, 223 objects/s recovering 2023-10-09T04:41:16.038 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[77063]: audit 2023-10-09T04:41:15.223235+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.82:0/740240655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.038 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[77063]: audit 2023-10-09T04:41:15.223861+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.038 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[79733]: cluster 2023-10-09T04:41:14.517308+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v834: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8106/58362 objects degraded (13.889%); 6.7 MiB/s, 223 objects/s recovering 2023-10-09T04:41:16.039 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[79733]: audit 2023-10-09T04:41:15.223235+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.82:0/740240655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.039 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:15 smithi082 bash[79733]: audit 2023-10-09T04:41:15.223861+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:16.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:16.173 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:15 smithi153 bash[59565]: cluster 2023-10-09T04:41:14.517308+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v834: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8106/58362 objects degraded (13.889%); 6.7 MiB/s, 223 objects/s recovering 2023-10-09T04:41:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:15 smithi153 bash[59565]: audit 2023-10-09T04:41:15.223235+0000 mon.c (mon.2) 86 : audit 1 from='client.? 172.21.15.82:0/740240655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:15 smithi153 bash[59565]: audit 2023-10-09T04:41:15.223861+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:16.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:41:16.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:16.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:16.432 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:16.583 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:16.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:16.720 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:16.749 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:16.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:16.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:16.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:16.999 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: cluster 2023-10-09T04:41:15.957377+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:16.999 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: cluster 2023-10-09T04:41:15.972159+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:16.999 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: cluster 2023-10-09T04:41:15.975859+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:16.999 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: audit 2023-10-09T04:41:15.976608+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:16.999 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: cluster 2023-10-09T04:41:15.981243+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:41:17.000 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: cluster 2023-10-09T04:41:15.988566+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.000 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: audit 2023-10-09T04:41:16.380329+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.000 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:16 smithi153 bash[59565]: audit 2023-10-09T04:41:16.391833+0000 mon.a (mon.0) 1195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:17.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: cluster 2023-10-09T04:41:15.957377+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: cluster 2023-10-09T04:41:15.972159+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: cluster 2023-10-09T04:41:15.975859+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: audit 2023-10-09T04:41:15.976608+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: cluster 2023-10-09T04:41:15.981243+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:41:17.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: cluster 2023-10-09T04:41:15.988566+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: audit 2023-10-09T04:41:16.380329+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[77063]: audit 2023-10-09T04:41:16.391833+0000 mon.a (mon.0) 1195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: cluster 2023-10-09T04:41:15.957377+0000 mon.a (mon.0) 1191 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: cluster 2023-10-09T04:41:15.972159+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: cluster 2023-10-09T04:41:15.975859+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: audit 2023-10-09T04:41:15.976608+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: cluster 2023-10-09T04:41:15.981243+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: cluster 2023-10-09T04:41:15.988566+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T04:41:17.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: audit 2023-10-09T04:41:16.380329+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:16 smithi082 bash[79733]: audit 2023-10-09T04:41:16.391833+0000 mon.a (mon.0) 1195 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:17.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:17.390 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:17.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:17.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:17.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:17.559 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:17.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:17.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:17.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:17.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:17.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:17.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:17.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:17.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:18.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.089 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:18.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:17 smithi153 bash[59565]: cluster 2023-10-09T04:41:16.518278+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 8106/57192 objects degraded (14.173%); 2.8 MiB/s, 192 objects/s recovering 2023-10-09T04:41:18.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:18.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:18.260 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:17 smithi082 bash[77063]: cluster 2023-10-09T04:41:16.518278+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 8106/57192 objects degraded (14.173%); 2.8 MiB/s, 192 objects/s recovering 2023-10-09T04:41:18.260 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:17 smithi082 bash[79733]: cluster 2023-10-09T04:41:16.518278+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 8106/57192 objects degraded (14.173%); 2.8 MiB/s, 192 objects/s recovering 2023-10-09T04:41:18.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:18.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:18.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:18.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:18.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:18.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:18.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:18.901 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:18.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:19.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:19.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:19.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.163 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:19.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:19.298 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:19.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:19.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[79733]: cluster 2023-10-09T04:41:18.394733+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[79733]: cluster 2023-10-09T04:41:18.397047+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:41:19.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[79733]: cluster 2023-10-09T04:41:18.403514+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[79733]: cluster 2023-10-09T04:41:18.518838+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1019 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7872/48759 objects degraded (16.145%); 5.2 MiB/s, 351 objects/s recovering 2023-10-09T04:41:19.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:19.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:19.674 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:19 smithi153 bash[59565]: cluster 2023-10-09T04:41:18.394733+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:19 smithi153 bash[59565]: cluster 2023-10-09T04:41:18.397047+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:41:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:19 smithi153 bash[59565]: cluster 2023-10-09T04:41:18.403514+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:19 smithi153 bash[59565]: cluster 2023-10-09T04:41:18.518838+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1019 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7872/48759 objects degraded (16.145%); 5.2 MiB/s, 351 objects/s recovering 2023-10-09T04:41:19.684 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[77063]: cluster 2023-10-09T04:41:18.394733+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.684 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[77063]: cluster 2023-10-09T04:41:18.397047+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:41:19.684 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[77063]: cluster 2023-10-09T04:41:18.403514+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:19.684 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[77063]: cluster 2023-10-09T04:41:18.518838+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 1019 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7872/48759 objects degraded (16.145%); 5.2 MiB/s, 351 objects/s recovering 2023-10-09T04:41:19.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:19.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:19.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:19.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:19.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.958 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:19] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-10-09T04:41:19.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:19.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:20.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:20.110 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:20.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:20.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:20.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:20.397 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:20.420 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.3 primary-affinity to 0.418784 (8274452) 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:18.395335+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.404316+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.405146+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.405482+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.406947+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.409507+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.482006+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.484592+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.498142+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: audit 2023-10-09T04:41:19.547817+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.82:0/1479324595' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: audit 2023-10-09T04:41:19.548532+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[77063]: cluster 2023-10-09T04:41:19.707082+0000 mon.a (mon.0) 1199 : cluster 3 Health check update: Degraded data redundancy: 7872/48759 objects degraded (16.145%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:18.395335+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.404316+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.405146+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.405482+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.406947+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.409507+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.482006+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.484592+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.498142+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: audit 2023-10-09T04:41:19.547817+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.82:0/1479324595' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: audit 2023-10-09T04:41:19.548532+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:20 smithi082 bash[79733]: cluster 2023-10-09T04:41:19.707082+0000 mon.a (mon.0) 1199 : cluster 3 Health check update: Degraded data redundancy: 7872/48759 objects degraded (16.145%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:20.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:20.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:20.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:18.395335+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T04:41:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.404316+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.405146+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.405482+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.406947+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.409507+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.482006+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.484592+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.498142+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: audit 2023-10-09T04:41:19.547817+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.82:0/1479324595' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: audit 2023-10-09T04:41:19.548532+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]: dispatch 2023-10-09T04:41:20.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:20 smithi153 bash[59565]: cluster 2023-10-09T04:41:19.707082+0000 mon.a (mon.0) 1199 : cluster 3 Health check update: Degraded data redundancy: 7872/48759 objects degraded (16.145%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:41:20.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:20.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:20.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:20.923 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:20.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:21.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:21.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:21.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:21.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:21.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:21.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:21.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:21.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:21.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:21.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:21.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.403617+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: audit 2023-10-09T04:41:20.411244+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]': finished 2023-10-09T04:41:21.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.411461+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:41:21.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.419970+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.422118+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.426587+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.428120+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.428637+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.519383+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 974 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7872/46605 objects degraded (16.891%); 3.2 MiB/s, 236 objects/s recovering 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[77063]: cluster 2023-10-09T04:41:20.572334+0000 osd.1 (osd.1) 1 : cluster 0 7.2 scrub starts 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.403617+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.558 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: audit 2023-10-09T04:41:20.411244+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]': finished 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.411461+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.419970+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.422118+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.426587+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.428120+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.428637+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.519383+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 974 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7872/46605 objects degraded (16.891%); 3.2 MiB/s, 236 objects/s recovering 2023-10-09T04:41:21.559 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:21 smithi082 bash[79733]: cluster 2023-10-09T04:41:20.572334+0000 osd.1 (osd.1) 1 : cluster 0 7.2 scrub starts 2023-10-09T04:41:21.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:21.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:21.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.403617+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: audit 2023-10-09T04:41:20.411244+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.41878403184908564}]': finished 2023-10-09T04:41:21.692 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.411461+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.419970+0000 osd.4 (osd.4) 150 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.422118+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.426587+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.428120+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.428637+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.519383+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 88 active+clean; 974 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7872/46605 objects degraded (16.891%); 3.2 MiB/s, 236 objects/s recovering 2023-10-09T04:41:21.693 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:21 smithi153 bash[59565]: cluster 2023-10-09T04:41:20.572334+0000 osd.1 (osd.1) 1 : cluster 0 7.2 scrub starts 2023-10-09T04:41:21.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:21.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:21.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:21.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:21.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:21.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:21.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:22.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:22.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:22.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:22.251 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:22.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:22.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:22.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:22.535 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:22.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.417641+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.419435+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.420446+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.421119+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.422020+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:41:22.648 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[77063]: cluster 2023-10-09T04:41:21.576433+0000 osd.1 (osd.1) 2 : cluster 0 7.12 scrub starts 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.417641+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.419435+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.420446+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.421119+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.422020+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:41:22.649 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:22 smithi082 bash[79733]: cluster 2023-10-09T04:41:21.576433+0000 osd.1 (osd.1) 2 : cluster 0 7.12 scrub starts 2023-10-09T04:41:22.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:22.697 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:22.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:22.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:22.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.417641+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.419435+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.420446+0000 osd.4 (osd.4) 151 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.421119+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T04:41:22.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.422020+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:41:22.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:22 smithi153 bash[59565]: cluster 2023-10-09T04:41:21.576433+0000 osd.1 (osd.1) 2 : cluster 0 7.12 scrub starts 2023-10-09T04:41:22.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:22.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:22.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:23.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:23.102 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:23.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:23.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:23.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:23.374 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:23.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:23 smithi082 bash[77063]: cluster 2023-10-09T04:41:22.520011+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 90 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 7103/44634 objects degraded (15.914%); 0 B/s, 130 objects/s recovering 2023-10-09T04:41:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:23 smithi082 bash[77063]: cluster 2023-10-09T04:41:22.608906+0000 osd.1 (osd.1) 3 : cluster 0 7.2 scrub starts 2023-10-09T04:41:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:23 smithi082 bash[79733]: cluster 2023-10-09T04:41:22.520011+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 90 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 7103/44634 objects degraded (15.914%); 0 B/s, 130 objects/s recovering 2023-10-09T04:41:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:23 smithi082 bash[79733]: cluster 2023-10-09T04:41:22.608906+0000 osd.1 (osd.1) 3 : cluster 0 7.2 scrub starts 2023-10-09T04:41:23.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:23.517 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:23.621 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:23.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:23.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:23.782 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:23.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:23.895 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:23 smithi153 bash[59565]: cluster 2023-10-09T04:41:22.520011+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 90 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 7103/44634 objects degraded (15.914%); 0 B/s, 130 objects/s recovering 2023-10-09T04:41:23.895 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:23 smithi153 bash[59565]: cluster 2023-10-09T04:41:22.608906+0000 osd.1 (osd.1) 3 : cluster 0 7.2 scrub starts 2023-10-09T04:41:23.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:23.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:24.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:24.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:24.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:24.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:24.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:24.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:24.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:24.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:24.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:24.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:24.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:24.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:24 smithi082 bash[79733]: cluster 2023-10-09T04:41:23.132693+0000 osd.5 (osd.5) 93 : cluster 0 7.0 scrub starts 2023-10-09T04:41:24.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:24 smithi082 bash[79733]: cluster 2023-10-09T04:41:23.601274+0000 osd.1 (osd.1) 4 : cluster 0 7.12 scrub starts 2023-10-09T04:41:24.699 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:24 smithi082 bash[77063]: cluster 2023-10-09T04:41:23.132693+0000 osd.5 (osd.5) 93 : cluster 0 7.0 scrub starts 2023-10-09T04:41:24.699 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:24 smithi082 bash[77063]: cluster 2023-10-09T04:41:23.601274+0000 osd.1 (osd.1) 4 : cluster 0 7.12 scrub starts 2023-10-09T04:41:24.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:24.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:24.784 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:24.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:24.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:24.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:24.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:24 smithi153 bash[59565]: cluster 2023-10-09T04:41:23.132693+0000 osd.5 (osd.5) 93 : cluster 0 7.0 scrub starts 2023-10-09T04:41:24.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:24 smithi153 bash[59565]: cluster 2023-10-09T04:41:23.601274+0000 osd.1 (osd.1) 4 : cluster 0 7.12 scrub starts 2023-10-09T04:41:24.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:25.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:25.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:25.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:25.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:25.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:25.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:25.389 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:25.417 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:25.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:25 smithi082 bash[79733]: cluster 2023-10-09T04:41:24.520864+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 828 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 6905/39618 objects degraded (17.429%); 2.1 MiB/s, 135 objects/s recovering 2023-10-09T04:41:25.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:25 smithi082 bash[79733]: cluster 2023-10-09T04:41:24.708402+0000 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 7103/44634 objects degraded (15.914%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:25.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:25.531 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:25 smithi153 bash[59565]: cluster 2023-10-09T04:41:24.520864+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 828 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 6905/39618 objects degraded (17.429%); 2.1 MiB/s, 135 objects/s recovering 2023-10-09T04:41:25.531 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:25 smithi153 bash[59565]: cluster 2023-10-09T04:41:24.708402+0000 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 7103/44634 objects degraded (15.914%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:25.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:25.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:25.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:25 smithi082 bash[77063]: cluster 2023-10-09T04:41:24.520864+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 828 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 6905/39618 objects degraded (17.429%); 2.1 MiB/s, 135 objects/s recovering 2023-10-09T04:41:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:25 smithi082 bash[77063]: cluster 2023-10-09T04:41:24.708402+0000 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 7103/44634 objects degraded (15.914%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:25.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:25.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:25.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:25.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:25.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:25.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:26.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:26.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:26.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:26.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:26.357 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:26.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:26.494 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:26.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:26.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:26.635 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:26 smithi153 bash[59565]: audit 2023-10-09T04:41:25.583292+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.635 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:26 smithi153 bash[59565]: audit 2023-10-09T04:41:25.593728+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:26 smithi153 bash[59565]: audit 2023-10-09T04:41:26.174866+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:26.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:26 smithi153 bash[59565]: audit 2023-10-09T04:41:26.176047+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:26.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:26 smithi153 bash[59565]: audit 2023-10-09T04:41:26.193384+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[77063]: audit 2023-10-09T04:41:25.583292+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[77063]: audit 2023-10-09T04:41:25.593728+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[77063]: audit 2023-10-09T04:41:26.174866+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[77063]: audit 2023-10-09T04:41:26.176047+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[77063]: audit 2023-10-09T04:41:26.193384+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[79733]: audit 2023-10-09T04:41:25.583292+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[79733]: audit 2023-10-09T04:41:25.593728+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[79733]: audit 2023-10-09T04:41:26.174866+0000 mon.a (mon.0) 1206 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[79733]: audit 2023-10-09T04:41:26.176047+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:41:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:26 smithi082 bash[79733]: audit 2023-10-09T04:41:26.193384+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:26.753 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:26.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:26.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:26.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:26.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:27.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:27.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:27.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:27.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:27.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:27.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:27.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:27.277 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:41:27.277 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:41:27.278 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:41:27.278 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:41:27.278 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:41:27.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:41:27.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:27.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:27.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:27.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:27.532 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:27.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:27.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:27.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:27.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:27.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:27.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.116496+0000 osd.5 (osd.5) 94 : cluster 0 7.0 scrub starts 2023-10-09T04:41:27.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.521548+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v845: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 789 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 6905/37755 objects degraded (18.289%); 1.7 MiB/s, 114 objects/s recovering 2023-10-09T04:41:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.594486+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.596574+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.597188+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:41:27.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:27 smithi153 bash[59565]: cluster 2023-10-09T04:41:26.614943+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.938 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:27.946 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.116496+0000 osd.5 (osd.5) 94 : cluster 0 7.0 scrub starts 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.521548+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v845: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 789 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 6905/37755 objects degraded (18.289%); 1.7 MiB/s, 114 objects/s recovering 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.594486+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.596574+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.597188+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[79733]: cluster 2023-10-09T04:41:26.614943+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.116496+0000 osd.5 (osd.5) 94 : cluster 0 7.0 scrub starts 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.521548+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v845: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 91 active+clean; 789 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 6905/37755 objects degraded (18.289%); 1.7 MiB/s, 114 objects/s recovering 2023-10-09T04:41:27.947 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.594486+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.948 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.596574+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.948 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.597188+0000 mon.a (mon.0) 1209 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:41:27.948 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:27 smithi082 bash[77063]: cluster 2023-10-09T04:41:26.614943+0000 osd.4 (osd.4) 152 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T04:41:27.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:27.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:28.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:28.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:28.199 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:28.212 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:28.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:28.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:28.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:28.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:28.492 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:28.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:28.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:28.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:28.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:28.758 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:28 smithi153 bash[59565]: cluster 2023-10-09T04:41:27.596727+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.758 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:28 smithi153 bash[59565]: cluster 2023-10-09T04:41:27.598362+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.758 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:28 smithi153 bash[59565]: cluster 2023-10-09T04:41:27.599896+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:41:28.759 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:28 smithi153 bash[59565]: cluster 2023-10-09T04:41:27.600623+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.759 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:28 smithi153 bash[59565]: cluster 2023-10-09T04:41:27.668839+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.774 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:28.876 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[77063]: cluster 2023-10-09T04:41:27.596727+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[77063]: cluster 2023-10-09T04:41:27.598362+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[77063]: cluster 2023-10-09T04:41:27.599896+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[77063]: cluster 2023-10-09T04:41:27.600623+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[77063]: cluster 2023-10-09T04:41:27.668839+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[79733]: cluster 2023-10-09T04:41:27.596727+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.885 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[79733]: cluster 2023-10-09T04:41:27.598362+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[79733]: cluster 2023-10-09T04:41:27.599896+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:41:28.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[79733]: cluster 2023-10-09T04:41:27.600623+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.886 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:28 smithi082 bash[79733]: cluster 2023-10-09T04:41:27.668839+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T04:41:28.901 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:28.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:29.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.037 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:29.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:29.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:29.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.306 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:29.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:29.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:29.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:29.534 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:29.564 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:29.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.672 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:29 smithi082 bash[77063]: cluster 2023-10-09T04:41:28.522068+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v848: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 646 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5959/30879 objects degraded (19.298%); 3.2 MiB/s, 206 objects/s recovering 2023-10-09T04:41:29.672 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:29 smithi082 bash[77063]: audit 2023-10-09T04:41:29.280064+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:29.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:29.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:29.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:29.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:29 smithi153 bash[59565]: cluster 2023-10-09T04:41:28.522068+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v848: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 646 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5959/30879 objects degraded (19.298%); 3.2 MiB/s, 206 objects/s recovering 2023-10-09T04:41:29.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:29 smithi153 bash[59565]: audit 2023-10-09T04:41:29.280064+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:29.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:29.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:29.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:29 smithi082 bash[79733]: cluster 2023-10-09T04:41:28.522068+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v848: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 646 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5959/30879 objects degraded (19.298%); 3.2 MiB/s, 206 objects/s recovering 2023-10-09T04:41:29.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:29 smithi082 bash[79733]: audit 2023-10-09T04:41:29.280064+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:29] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T04:41:30.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:30.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:30.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:30.218 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:30.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:30.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:30.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:30.485 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:30.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:30.623 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:30.650 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:30.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:30.780 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:30.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:30.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:30 smithi082 bash[77063]: cluster 2023-10-09T04:41:29.710046+0000 mon.a (mon.0) 1212 : cluster 3 Health check update: Degraded data redundancy: 5959/30879 objects degraded (19.298%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:30.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:30 smithi082 bash[79733]: cluster 2023-10-09T04:41:29.710046+0000 mon.a (mon.0) 1212 : cluster 3 Health check update: Degraded data redundancy: 5959/30879 objects degraded (19.298%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:30.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:30.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:30 smithi153 bash[59565]: cluster 2023-10-09T04:41:29.710046+0000 mon.a (mon.0) 1212 : cluster 3 Health check update: Degraded data redundancy: 5959/30879 objects degraded (19.298%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:31.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:31.158 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:31.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:31.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:31.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:31.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:31.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:31.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:31.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:31 smithi153 bash[59565]: cluster 2023-10-09T04:41:30.036339+0000 osd.5 (osd.5) 97 : cluster 0 7.0 scrub starts 2023-10-09T04:41:31.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:31 smithi153 bash[59565]: cluster 2023-10-09T04:41:30.522715+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v849: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 598 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5959/28578 objects degraded (20.852%); 2.8 MiB/s, 124 objects/s recovering 2023-10-09T04:41:31.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:31.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:31.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:31.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:31 smithi082 bash[77063]: cluster 2023-10-09T04:41:30.036339+0000 osd.5 (osd.5) 97 : cluster 0 7.0 scrub starts 2023-10-09T04:41:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:31 smithi082 bash[77063]: cluster 2023-10-09T04:41:30.522715+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v849: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 598 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5959/28578 objects degraded (20.852%); 2.8 MiB/s, 124 objects/s recovering 2023-10-09T04:41:31.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:31 smithi082 bash[79733]: cluster 2023-10-09T04:41:30.036339+0000 osd.5 (osd.5) 97 : cluster 0 7.0 scrub starts 2023-10-09T04:41:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:31 smithi082 bash[79733]: cluster 2023-10-09T04:41:30.522715+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v849: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 598 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5959/28578 objects degraded (20.852%); 2.8 MiB/s, 124 objects/s recovering 2023-10-09T04:41:31.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:31.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:32.061 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:32.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:32.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:32.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:32.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:32.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:32.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:32.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:32.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:32.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:32.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:32.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:32.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:32.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:32.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:32.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:32.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:32.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:33.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:33.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:33.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:33.159 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:33.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:33.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:33.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:33.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:33.534 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:33.562 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:33.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:33.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:33.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:33.809 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:33 smithi082 bash[77063]: cluster 2023-10-09T04:41:32.523268+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v850: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5959/25797 objects degraded (23.100%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T04:41:33.809 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:33 smithi082 bash[77063]: cluster 2023-10-09T04:41:32.964714+0000 osd.5 (osd.5) 98 : cluster 0 7.0 scrub starts 2023-10-09T04:41:33.809 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:33 smithi082 bash[79733]: cluster 2023-10-09T04:41:32.523268+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v850: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5959/25797 objects degraded (23.100%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T04:41:33.809 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:33 smithi082 bash[79733]: cluster 2023-10-09T04:41:32.964714+0000 osd.5 (osd.5) 98 : cluster 0 7.0 scrub starts 2023-10-09T04:41:33.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:33.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:33 smithi153 bash[59565]: cluster 2023-10-09T04:41:32.523268+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v850: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5959/25797 objects degraded (23.100%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T04:41:33.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:33 smithi153 bash[59565]: cluster 2023-10-09T04:41:32.964714+0000 osd.5 (osd.5) 98 : cluster 0 7.0 scrub starts 2023-10-09T04:41:33.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:33.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:33.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:34.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:34.211 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.239 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:34.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:34.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87206 6.c 6.51e6993c (undecoded) ondisk+retry+write+known_if_redirected e139)", 2023-10-09T04:41:34.403 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 504.64864612500003, 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.019224345, 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:41:34.404 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87206 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.405 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742805+0000", 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.406 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742804+0000", 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742829+0000", 2023-10-09T04:41:34.407 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5610999999999999e-05 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742837+0000", 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0819999999999999e-06 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.408 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742845+0000", 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2069999999999998e-06 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742889+0000", 2023-10-09T04:41:34.409 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4777000000000002e-05 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op must wait for map", 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.742894+0000", 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8300000000000003e-06 2023-10-09T04:41:34.410 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.755779+0000", 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012884678 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.411 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.755788+0000", 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8890000000000008e-06 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.761987+0000", 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0061995909999999 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.412 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762029+0000", 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1962999999999998e-05 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.413 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87233 6.c 6.8e484a9c (undecoded) ondisk+write+known_if_redirected e139)", 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 504.61818696500001, 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98904841899999996, 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.414 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87233 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:41:34.415 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773264+0000", 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773262+0000", 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773303+0000", 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0358000000000002e-05 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773320+0000", 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6753999999999999e-05 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.417 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773328+0000", 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773415+0000", 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7342000000000005e-05 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.418 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.773432+0000", 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7217000000000001e-05 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762301+0000", 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98886854599999996 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.419 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762312+0000", 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1591e-05 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.420 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.34995.0:87251 6.c 6.ab78b1bc (undecoded) ondisk+write+known_if_redirected e139)", 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 504.59064698999998, 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96155623000000001, 2023-10-09T04:41:34.421 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.34995", 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 87251 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.422 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.423 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800804+0000", 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800802+0000", 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.424 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800875+0000", 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3133999999999995e-05 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.425 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800911+0000", 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6018e-05 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.800964+0000", 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2862999999999999e-05 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.426 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.801027+0000", 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3337999999999996e-05 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:09.801042+0000", 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5002e-05 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762351+0000", 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96130837400000002 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:33:10.762360+0000", 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3950000000000007e-06 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.428 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35259.0:153439 7.8 e193/190 7:1679e27e:::benchmark_data_smithi082_155217_object50779:head v 193'11322, mlcod=182'6525)", 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:25.560941+0000", 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 68.830510191000002, 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12554659800000001, 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.429 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560941+0000", 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560941+0000", 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.430 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560940+0000", 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560944+0000", 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8940000000000003e-06 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560951+0000", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.562e-06 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.560956+0000", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7360000000000001e-06 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.658498+0000", 2023-10-09T04:41:34.432 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.097541883999999995 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.658525+0000", 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7412000000000001e-05 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.686457+0000", 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027932512999999999 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:25.686487+0000", 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9949000000000001e-05 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191125 7.3 7:c6bdfaff:::benchmark_data_smithi082_155217_object88465:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e206)", 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.418523+0000", 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.972927693000001, 2023-10-09T04:41:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.057565992, 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191125 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.435 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418523+0000", 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418523+0000", 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.436 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418522+0000", 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418528+0000", 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9460000000000003e-06 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418536+0000", 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8699999999999998e-06 2023-10-09T04:41:34.438 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418545+0000", 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2290000000000003e-06 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.439 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.419600+0000", 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0010557209999999999 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.419611+0000", 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.102e-05 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.440 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.473410+0000", 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.053798453 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.441 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476089+0000", 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002679467 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.442 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191135 7.13 7:cbc40fc3:::benchmark_data_smithi082_155217_object88475:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e206)", 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.418556+0000", 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.972895295000001, 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.033134606, 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191135 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.443 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418556+0000", 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418556+0000", 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.444 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418555+0000", 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418558+0000", 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4449999999999999e-06 2023-10-09T04:41:34.445 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418563+0000", 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7540000000000003e-06 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.446 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.418568+0000", 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9649999999999999e-06 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.419637+0000", 2023-10-09T04:41:34.447 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001068666 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.419645+0000", 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3599999999999996e-06 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.448 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.451590+0000", 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.031944408 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.451690+0000", 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000100436 2023-10-09T04:41:34.449 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191151 7.13 7:cca8fdd0:::benchmark_data_smithi082_155217_object88491:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.447452+0000", 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.943999166999999, 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.015334178, 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.450 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191151 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447452+0000", 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.451 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447452+0000", 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.452 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447449+0000", 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447460+0000", 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.046e-05 2023-10-09T04:41:34.453 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447483+0000", 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3349e-05 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.454 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447495+0000", 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1664999999999999e-05 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447727+0000", 2023-10-09T04:41:34.455 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00023172899999999999 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.447738+0000", 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1705000000000001e-05 2023-10-09T04:41:34.456 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.452431+0000", 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0046926949999999 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.457 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.452539+0000", 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000108089 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.453017+0000", 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000477806 2023-10-09T04:41:34.458 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462610+0000", 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0095928560000000003 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.459 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462678+0000", 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8238000000000006e-05 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.460 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462727+0000", 2023-10-09T04:41:34.461 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8686000000000001e-05 2023-10-09T04:41:34.461 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.461 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.461 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.461 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462760+0000", 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3540000000000001e-05 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462786+0000", 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5777000000000001e-05 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.462 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191163 7.3 7:c0dc68b9:::benchmark_data_smithi082_155217_object88503:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.462237+0000", 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.929214010999999, 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0348778030000001, 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191163 2023-10-09T04:41:34.463 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462237+0000", 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462237+0000", 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462235+0000", 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462243+0000", 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2359999999999998e-06 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.465 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462254+0000", 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1255e-05 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462260+0000", 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4350000000000002e-06 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.466 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462373+0000", 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000112983 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.462385+0000", 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2012e-05 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476125+0000", 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.013739741 2023-10-09T04:41:34.467 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476243+0000", 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011825800000000001 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476756+0000", 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00051304400000000004 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.468 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486202+0000", 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0094458149999999998 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.496877+0000", 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.01067474 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.469 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497032+0000", 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000154971 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497058+0000", 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5786999999999999e-05 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.470 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497115+0000", 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7095999999999997e-05 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191182 7.13 7:c925f57d:::benchmark_data_smithi082_155217_object88522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.488380+0000", 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.903070811999999, 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.97899390900000005, 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.471 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191182 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488380+0000", 2023-10-09T04:41:34.472 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488380+0000", 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488378+0000", 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488386+0000", 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3969999999999997e-06 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488404+0000", 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8570999999999999e-05 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.474 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488413+0000", 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7499999999999992e-06 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488475+0000", 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2045000000000007e-05 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.475 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.488487+0000", 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1674999999999999e-05 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.455711+0000", 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96722382399999995 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.476 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.455981+0000", 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027032100000000002 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.456670+0000", 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00068855700000000001 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462720+0000", 2023-10-09T04:41:34.477 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060508009999999997 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462792+0000", 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1221000000000001e-05 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467312+0000", 2023-10-09T04:41:34.478 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0045198440000000003 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467346+0000", 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4646e-05 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467374+0000", 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7962000000000001e-05 2023-10-09T04:41:34.479 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191185 7.3 7:c6db0903:::benchmark_data_smithi082_155217_object88525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.490657+0000", 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.900793817, 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0071199470000001, 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.480 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191185 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490657+0000", 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.481 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490657+0000", 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490655+0000", 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.482 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490663+0000", 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9559999999999997e-06 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490801+0000", 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013814 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.483 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490814+0000", 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2673e-05 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490881+0000", 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6752000000000001e-05 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.490894+0000", 2023-10-09T04:41:34.484 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2665e-05 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479670+0000", 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98877614999999996 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479819+0000", 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014967100000000001 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.485 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.480383+0000", 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00056335300000000005 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486208+0000", 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0058251630000000004 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.486 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497161+0000", 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010953519 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497700+0000", 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00053852100000000003 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.487 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497733+0000", 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.2922000000000001e-05 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497777+0000", 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4279000000000001e-05 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.488 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191192 7.3 7:c0c07c74:::benchmark_data_smithi082_155217_object88532:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.504799+0000", 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.886652270999999, 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.99312152499999995, 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191192 2023-10-09T04:41:34.489 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504799+0000", 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504799+0000", 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.490 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504797+0000", 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504806+0000", 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1400000000000006e-06 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.491 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504823+0000", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7051e-05 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504834+0000", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.029e-05 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504906+0000", 2023-10-09T04:41:34.492 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2259000000000001e-05 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.504928+0000", 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1787e-05 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.482950+0000", 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.97802209799999995 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.493 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.483080+0000", 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013058199999999999 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.483523+0000", 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00044243 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.494 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.496885+0000", 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013362687999999999 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497622+0000", 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00073636800000000005 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497831+0000", 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020921000000000001 2023-10-09T04:41:34.495 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497873+0000", 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2518999999999997e-05 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497920+0000", 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6828e-05 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.496 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191201 7.13 7:cebd97e4:::benchmark_data_smithi082_155217_object88541:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.518000+0000", 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.873450574, 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94953583100000005, 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191201 2023-10-09T04:41:34.497 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518000+0000", 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518000+0000", 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.498 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.517999+0000", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518008+0000", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1309999999999995e-06 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518022+0000", 2023-10-09T04:41:34.499 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4139e-05 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518031+0000", 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.952e-06 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518091+0000", 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0531e-05 2023-10-09T04:41:34.500 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.518100+0000", 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8599999999999999e-06 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.459977+0000", 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94187657300000005 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.501 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.460105+0000", 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012814900000000001 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.460502+0000", 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00039747799999999999 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467213+0000", 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0067110549999999996 2023-10-09T04:41:34.502 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467253+0000", 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0092999999999997e-05 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467438+0000", 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000184029 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.503 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467485+0000", 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7163999999999998e-05 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467536+0000", 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1606e-05 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.504 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191210 7.13 7:cfff5578:::benchmark_data_smithi082_155217_object88550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.529204+0000", 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.862246614, 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.93853997600000005, 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191210 2023-10-09T04:41:34.505 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529204+0000", 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529204+0000", 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.506 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529203+0000", 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529209+0000", 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5740000000000001e-06 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.507 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529222+0000", 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2367e-05 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529230+0000", 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2989999999999997e-06 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529309+0000", 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8883999999999999e-05 2023-10-09T04:41:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529317+0000", 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.729e-06 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462811+0000", 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.93349457999999996 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.509 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.462935+0000", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000124146 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.463224+0000", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00028904899999999999 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467263+0000", 2023-10-09T04:41:34.510 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040383670000000002 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467596+0000", 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000332765 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467654+0000", 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8649e-05 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.511 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467701+0000", 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6319000000000002e-05 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467744+0000", 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3877e-05 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.512 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191211 7.13 7:ce729b32:::benchmark_data_smithi082_155217_object88551:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.529251+0000", 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.862200040999999, 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94149622600000005, 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.513 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191211 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529251+0000", 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529251+0000", 2023-10-09T04:41:34.514 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529250+0000", 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529254+0000", 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.579e-06 2023-10-09T04:41:34.515 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529262+0000", 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6159999999999996e-06 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529267+0000", 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7690000000000002e-06 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.516 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529330+0000", 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2974999999999999e-05 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.529335+0000", 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.279e-06 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.517 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.465515+0000", 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.93617990799999995 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.465650+0000", 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000135604 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.518 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.465950+0000", 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00029994899999999998 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467763+0000", 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0018125699999999999 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470620+0000", 2023-10-09T04:41:34.519 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028578319999999998 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470655+0000", 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4227999999999998e-05 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470707+0000", 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2323e-05 2023-10-09T04:41:34.520 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470747+0000", 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0166999999999997e-05 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191234 7.3 7:c7aa3d18:::benchmark_data_smithi082_155217_object88574:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.572068+0000", 2023-10-09T04:41:34.521 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.819383444, 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.95256344999999998, 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191234 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.522 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572068+0000", 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572068+0000", 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.523 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572066+0000", 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572075+0000", 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9339999999999995e-06 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.524 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572101+0000", 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6584000000000002e-05 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572108+0000", 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0079999999999996e-06 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572182+0000", 2023-10-09T04:41:34.525 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4159000000000006e-05 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.572196+0000", 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3464e-05 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486337+0000", 2023-10-09T04:41:34.526 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.91414141699999996 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486456+0000", 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000118162 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486942+0000", 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00048624300000000003 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.527 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497124+0000", 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010182402 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497673+0000", 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00054930799999999998 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.528 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524539+0000", 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.026865947000000001 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524588+0000", 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8297999999999997e-05 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.529 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524631+0000", 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3291000000000003e-05 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191248 7.13 7:cff45528:::benchmark_data_smithi082_155217_object88588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.592249+0000", 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.799202104000001, 2023-10-09T04:41:34.530 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.89394267500000002, 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191248 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.531 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592249+0000", 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592249+0000", 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592246+0000", 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:41:34.532 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592257+0000", 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0473e-05 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592279+0000", 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2359000000000001e-05 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.533 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592292+0000", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.27e-05 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592369+0000", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7055e-05 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.592384+0000", 2023-10-09T04:41:34.534 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4997000000000001e-05 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467793+0000", 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.87540907999999995 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.467923+0000", 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000129362 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.535 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.468242+0000", 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031902999999999999 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476085+0000", 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0078431549999999992 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.536 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.476153+0000", 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8549999999999999e-05 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486103+0000", 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0099501420000000004 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486152+0000", 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8952999999999997e-05 2023-10-09T04:41:34.537 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486192+0000", 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9239999999999997e-05 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191250 7.3 7:c49f902c:::benchmark_data_smithi082_155217_object88590:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.538 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.598875+0000", 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.792576330999999, 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.92588247999999995, 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191250 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.539 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598875+0000", 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598875+0000", 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.540 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598873+0000", 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598883+0000", 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8439999999999993e-06 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598902+0000", 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8947e-05 2023-10-09T04:41:34.541 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598914+0000", 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.269e-05 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598972+0000", 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7998999999999998e-05 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.542 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.598985+0000", 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2249000000000001e-05 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497965+0000", 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.89898065000000005 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.543 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.498056+0000", 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0533999999999998e-05 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.498587+0000", 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00053176899999999997 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.544 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.505037+0000", 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064493509999999999 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.507938+0000", 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002901004 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.545 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524682+0000", 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.016744457000000001 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524720+0000", 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7685000000000003e-05 2023-10-09T04:41:34.546 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524757+0000", 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7181000000000002e-05 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.547 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191268 7.13 7:cd494a54:::benchmark_data_smithi082_155217_object88608:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.632058+0000", 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.759392664, 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.85425119699999996, 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.548 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191268 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632058+0000", 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.549 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632058+0000", 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632056+0000", 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.550 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632066+0000", 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6500000000000008e-06 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632081+0000", 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5128e-05 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.551 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632091+0000", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8679999999999994e-06 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632175+0000", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4284999999999997e-05 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.632193+0000", 2023-10-09T04:41:34.552 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7499999999999998e-05 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470782+0000", 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.83858877799999998 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.470903+0000", 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012112699999999999 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.553 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.471253+0000", 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000349976 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479641+0000", 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0083881059999999993 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.554 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479695+0000", 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3816000000000001e-05 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486239+0000", 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065444429999999996 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.555 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486276+0000", 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7169000000000003e-05 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.486310+0000", 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3288999999999999e-05 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.556 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191272 7.3 7:c109009f:::benchmark_data_smithi082_155217_object88612:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.643695+0000", 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.747755544, 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.88124905799999997, 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191272 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.557 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643695+0000", 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643695+0000", 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.558 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643693+0000", 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643705+0000", 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1634000000000001e-05 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643731+0000", 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5788000000000001e-05 2023-10-09T04:41:34.559 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643741+0000", 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.007e-05 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643811+0000", 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9822999999999999e-05 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.560 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.643824+0000", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3076e-05 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.501899+0000", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.85807491499999999 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.502092+0000", 2023-10-09T04:41:34.561 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000193232 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.502625+0000", 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00053336900000000001 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.507951+0000", 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053259500000000003 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.562 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.511492+0000", 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0035407669999999998 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524884+0000", 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013392585 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.563 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524906+0000", 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1339999999999999e-05 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.524945+0000", 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8782999999999998e-05 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:191291 7.13 7:cccd70be:::benchmark_data_smithi082_155217_object88631:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-09T04:41:34.564 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:20.752339+0000", 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 13.639111527000001, 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.74466215099999999, 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 191291 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:41:34.565 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752339+0000", 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752339+0000", 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.566 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752338+0000", 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752347+0000", 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7999999999999993e-06 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:41:34.567 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752365+0000", 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7107000000000001e-05 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752381+0000", 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6852000000000001e-05 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752482+0000", 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010091599999999999 2023-10-09T04:41:34.568 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:20.752496+0000", 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3505e-05 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.473455+0000", 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.72095898999999997 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.569 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.473679+0000", 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000224012 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.474088+0000", 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040922300000000001 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.570 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479703+0000", 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056152989999999998 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.479734+0000", 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0598000000000001e-05 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:34.571 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.496919+0000", 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017185496000000001 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.496964+0000", 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4895999999999998e-05 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:21.497002+0000", 2023-10-09T04:41:34.572 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7289999999999997e-05 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:41:34.573 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:41:34.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:34.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:34.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:34.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:34.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:34.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:34.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:34.984 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:34.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:35.088 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:35.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:35.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:35.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:35.370 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:35.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:35.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:35.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:35.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:35.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:35 smithi082 bash[77063]: cluster 2023-10-09T04:41:34.523962+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 398 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5959/18984 objects degraded (31.390%); 1.3 MiB/s, 137 objects/s recovering 2023-10-09T04:41:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:35 smithi082 bash[77063]: cluster 2023-10-09T04:41:34.711290+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5959/18984 objects degraded (31.390%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:35 smithi082 bash[79733]: cluster 2023-10-09T04:41:34.523962+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 398 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5959/18984 objects degraded (31.390%); 1.3 MiB/s, 137 objects/s recovering 2023-10-09T04:41:35.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:35 smithi082 bash[79733]: cluster 2023-10-09T04:41:34.711290+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5959/18984 objects degraded (31.390%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.784 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:35.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:35.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:35.915 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:35.922 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:35 smithi153 bash[59565]: cluster 2023-10-09T04:41:34.523962+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 398 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5959/18984 objects degraded (31.390%); 1.3 MiB/s, 137 objects/s recovering 2023-10-09T04:41:35.922 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:35 smithi153 bash[59565]: cluster 2023-10-09T04:41:34.711290+0000 mon.a (mon.0) 1213 : cluster 3 Health check update: Degraded data redundancy: 5959/18984 objects degraded (31.390%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:35.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:36.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:36.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:36.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:36.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:36.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:36.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:36.335 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:36.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:36.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:36.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:36.595 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:36.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:36.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:36.726 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[77063]: cluster 2023-10-09T04:41:35.717321+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[77063]: cluster 2023-10-09T04:41:35.722891+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[77063]: cluster 2023-10-09T04:41:35.725129+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:41:36.727 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[77063]: cluster 2023-10-09T04:41:35.729153+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:36.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:36.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:36.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:36.967 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:36 smithi153 bash[59565]: cluster 2023-10-09T04:41:35.717321+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.967 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:36 smithi153 bash[59565]: cluster 2023-10-09T04:41:35.722891+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.967 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:36 smithi153 bash[59565]: cluster 2023-10-09T04:41:35.725129+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:41:36.967 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:36 smithi153 bash[59565]: cluster 2023-10-09T04:41:35.729153+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:36.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:36.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:37.018 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[79733]: cluster 2023-10-09T04:41:35.717321+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:37.018 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[79733]: cluster 2023-10-09T04:41:35.722891+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:37.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[79733]: cluster 2023-10-09T04:41:35.725129+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:41:37.019 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:36 smithi082 bash[79733]: cluster 2023-10-09T04:41:35.729153+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T04:41:37.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:37.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:37.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:37.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:37.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:37.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:37.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:41:37.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:37.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:37.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:37.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:37.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:37.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.524460+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v853: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 354 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5959/16872 objects degraded (35.319%); 1.1 MiB/s, 89 objects/s recovering 2023-10-09T04:41:37.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.725736+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.890 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.729099+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.730600+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.736549+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[77063]: cluster 2023-10-09T04:41:36.741388+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.524460+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v853: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 354 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5959/16872 objects degraded (35.319%); 1.1 MiB/s, 89 objects/s recovering 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.725736+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.729099+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.730600+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.891 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.736549+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.892 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:37 smithi082 bash[79733]: cluster 2023-10-09T04:41:36.741388+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:37.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:37.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:38.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:38.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:38.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.524460+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v853: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 93 active+clean; 354 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 5959/16872 objects degraded (35.319%); 1.1 MiB/s, 89 objects/s recovering 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.725736+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.729099+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.730600+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.736549+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:37 smithi153 bash[59565]: cluster 2023-10-09T04:41:36.741388+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T04:41:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:38.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:38.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:38.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:38.450 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:41:38.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:38.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:38.493 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:41:38.547 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:41:38.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:38.586 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:41:38.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:38.625 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:41:38.664 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:41:38.693 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:41:38.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:38.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:38.741 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:41:38.744 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.744 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.745 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:41:38.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:38.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:38.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:38.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.007 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:38 smithi082 bash[77063]: audit 2023-10-09T04:41:38.385181+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.82:0/202118002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:39.007 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:38 smithi082 bash[79733]: audit 2023-10-09T04:41:38.385181+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.82:0/202118002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:39.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:39.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:39.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:39.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:38 smithi153 bash[59565]: audit 2023-10-09T04:41:38.385181+0000 mon.a (mon.0) 1216 : audit 0 from='client.? 172.21.15.82:0/202118002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:39.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:39.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:39.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:39.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:39.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:39.668 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:39.753 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:41:39.753 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "" 2023-10-09T04:41:39.753 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:41:39.759 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:41:39.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:39.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:39.927 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:39.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[77063]: cluster 2023-10-09T04:41:38.525180+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 203 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5142/9633 objects degraded (53.379%); 0 B/s, 96 objects/s recovering 2023-10-09T04:41:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[77063]: audit 2023-10-09T04:41:38.818001+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.82:0/3578716862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:39.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[77063]: audit 2023-10-09T04:41:38.818995+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.712689+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: Degraded data redundancy: 5142/9633 objects degraded (53.379%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[79733]: cluster 2023-10-09T04:41:38.525180+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 203 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5142/9633 objects degraded (53.379%); 0 B/s, 96 objects/s recovering 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[79733]: audit 2023-10-09T04:41:38.818001+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.82:0/3578716862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[79733]: audit 2023-10-09T04:41:38.818995+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.712689+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: Degraded data redundancy: 5142/9633 objects degraded (53.379%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:39] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T04:41:40.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:40.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:40.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:39 smithi153 bash[59565]: cluster 2023-10-09T04:41:38.525180+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v855: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 203 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5142/9633 objects degraded (53.379%); 0 B/s, 96 objects/s recovering 2023-10-09T04:41:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:39 smithi153 bash[59565]: audit 2023-10-09T04:41:38.818001+0000 mon.c (mon.2) 88 : audit 1 from='client.? 172.21.15.82:0/3578716862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:39 smithi153 bash[59565]: audit 2023-10-09T04:41:38.818995+0000 mon.a (mon.0) 1217 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:40.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:39 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.712689+0000 mon.a (mon.0) 1218 : cluster 3 Health check update: Degraded data redundancy: 5142/9633 objects degraded (53.379%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:40.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:40.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:40.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:40.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:40.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:40.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:40.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:40.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:40.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:40.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:40.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:40.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:40.979 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.736093+0000 mon.a (mon.0) 1219 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.742978+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: audit 2023-10-09T04:41:39.745743+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.745853+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.745935+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.752945+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.754130+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.759903+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[77063]: cluster 2023-10-09T04:41:39.951138+0000 osd.7 (osd.7) 150 : cluster 0 7.5 scrub starts 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.736093+0000 mon.a (mon.0) 1219 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:40.980 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.742978+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: audit 2023-10-09T04:41:39.745743+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.745853+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.745935+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.752945+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.754130+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.759903+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:40.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:40 smithi082 bash[79733]: cluster 2023-10-09T04:41:39.951138+0000 osd.7 (osd.7) 150 : cluster 0 7.5 scrub starts 2023-10-09T04:41:41.003 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:41.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.736093+0000 mon.a (mon.0) 1219 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:41.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.742978+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:41.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: audit 2023-10-09T04:41:39.745743+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:41.121 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.745853+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:41:41.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.745935+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:41.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.752945+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:41.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.754130+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:41.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.759903+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T04:41:41.122 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:40 smithi153 bash[59565]: cluster 2023-10-09T04:41:39.951138+0000 osd.7 (osd.7) 150 : cluster 0 7.5 scrub starts 2023-10-09T04:41:41.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:41.244 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 102657 objects 2023-10-09T04:41:41.245 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :154.845 2023-10-09T04:41:41.252 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:41.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:41.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:41.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:41.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:41.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:41.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:41.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:41.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:41 smithi082 bash[77063]: cluster 2023-10-09T04:41:40.525803+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 148 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5142/6975 objects degraded (73.720%); 0 B/s, 65 objects/s recovering 2023-10-09T04:41:41.938 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:41 smithi082 bash[77063]: cluster 2023-10-09T04:41:40.958295+0000 osd.7 (osd.7) 151 : cluster 0 7.5 scrub starts 2023-10-09T04:41:41.938 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:41 smithi082 bash[79733]: cluster 2023-10-09T04:41:40.525803+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 148 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5142/6975 objects degraded (73.720%); 0 B/s, 65 objects/s recovering 2023-10-09T04:41:41.938 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:41 smithi082 bash[79733]: cluster 2023-10-09T04:41:40.958295+0000 osd.7 (osd.7) 151 : cluster 0 7.5 scrub starts 2023-10-09T04:41:41.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:41.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:42.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:41 smithi153 bash[59565]: cluster 2023-10-09T04:41:40.525803+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v857: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 148 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5142/6975 objects degraded (73.720%); 0 B/s, 65 objects/s recovering 2023-10-09T04:41:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:41 smithi153 bash[59565]: cluster 2023-10-09T04:41:40.958295+0000 osd.7 (osd.7) 151 : cluster 0 7.5 scrub starts 2023-10-09T04:41:42.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.223 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:42.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:42.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:41:42.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:42.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:42.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:42.621 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:42.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:42.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:42.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:42.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:42.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:42.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:43.032 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:43.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:43.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:41:43.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.282 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:43.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:43.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:43.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:43.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:43.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:43.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:43.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:43.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:43.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:43.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:43 smithi153 bash[59565]: cluster 2023-10-09T04:41:42.526631+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 101 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5142/4698 objects degraded (109.451%); 0 B/s, 57 objects/s recovering 2023-10-09T04:41:43.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:43.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:43.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:43.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:43 smithi082 bash[77063]: cluster 2023-10-09T04:41:42.526631+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 101 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5142/4698 objects degraded (109.451%); 0 B/s, 57 objects/s recovering 2023-10-09T04:41:43.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:43 smithi082 bash[79733]: cluster 2023-10-09T04:41:42.526631+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v858: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 101 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 5142/4698 objects degraded (109.451%); 0 B/s, 57 objects/s recovering 2023-10-09T04:41:44.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:44.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:44.213 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:44.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:44.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:44.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:44.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:44.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:44.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:44.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:44.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:44.736 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:44.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:44.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:44.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:44.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:45.007 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:45.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:45.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:45.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:45.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.099290+0000 osd.6 (osd.6) 69 : cluster 0 7.7 scrub starts 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: audit 2023-10-09T04:41:44.287637+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: audit 2023-10-09T04:41:44.288925+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.339492+0000 osd.3 (osd.3) 1 : cluster 0 7.3 deep-scrub starts 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.355242+0000 osd.3 (osd.3) 2 : cluster 0 7.3 deep-scrub ok 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.527765+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 3573/1134 objects degraded (315.079%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:45.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:45 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.713976+0000 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 5142/4698 objects degraded (109.451%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:45.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.099290+0000 osd.6 (osd.6) 69 : cluster 0 7.7 scrub starts 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: audit 2023-10-09T04:41:44.287637+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: audit 2023-10-09T04:41:44.288925+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.339492+0000 osd.3 (osd.3) 1 : cluster 0 7.3 deep-scrub starts 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.355242+0000 osd.3 (osd.3) 2 : cluster 0 7.3 deep-scrub ok 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.527765+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 3573/1134 objects degraded (315.079%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.713976+0000 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 5142/4698 objects degraded (109.451%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.099290+0000 osd.6 (osd.6) 69 : cluster 0 7.7 scrub starts 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: audit 2023-10-09T04:41:44.287637+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:41:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: audit 2023-10-09T04:41:44.288925+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.339492+0000 osd.3 (osd.3) 1 : cluster 0 7.3 deep-scrub starts 2023-10-09T04:41:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.355242+0000 osd.3 (osd.3) 2 : cluster 0 7.3 deep-scrub ok 2023-10-09T04:41:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.527765+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v859: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 3573/1134 objects degraded (315.079%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:45.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:45 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.713976+0000 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 5142/4698 objects degraded (109.451%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:45.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:45.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:45.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:45.772 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:41:45.772 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:41:45.772 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:41:45.772 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:41:45.772 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:41:45.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:41:45.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:45.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:45.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:45.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:45.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:46.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:46.087 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:46.191 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:46.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:46.279 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T04:41:46.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T04:41:46.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:46.335 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:46 smithi153 bash[59565]: cluster 2023-10-09T04:41:44.758194+0000 osd.5 (osd.5) 102 : cluster 0 7.0 scrub starts 2023-10-09T04:41:46.358 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:46.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:46.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:46.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:46.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:46.603 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:46 smithi082 bash[77063]: cluster 2023-10-09T04:41:44.758194+0000 osd.5 (osd.5) 102 : cluster 0 7.0 scrub starts 2023-10-09T04:41:46.604 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:46 smithi082 bash[79733]: cluster 2023-10-09T04:41:44.758194+0000 osd.5 (osd.5) 102 : cluster 0 7.0 scrub starts 2023-10-09T04:41:46.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:46.722 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:46.754 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:46.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:46.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:46.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:46.993 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.013 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:47.034 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:47.034 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:47.117 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:47.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:47.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:47.180 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:47.181 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:47.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:47.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:47.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[79733]: cluster 2023-10-09T04:41:45.789700+0000 osd.5 (osd.5) 103 : cluster 0 7.0 scrub starts 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[79733]: cluster 2023-10-09T04:41:46.425286+0000 osd.3 (osd.3) 3 : cluster 0 7.13 scrub starts 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[79733]: cluster 2023-10-09T04:41:46.440283+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub ok 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[79733]: cluster 2023-10-09T04:41:46.528746+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 105 objects/s recovering 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[77063]: cluster 2023-10-09T04:41:45.789700+0000 osd.5 (osd.5) 103 : cluster 0 7.0 scrub starts 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[77063]: cluster 2023-10-09T04:41:46.425286+0000 osd.3 (osd.3) 3 : cluster 0 7.13 scrub starts 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[77063]: cluster 2023-10-09T04:41:46.440283+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub ok 2023-10-09T04:41:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:47 smithi082 bash[77063]: cluster 2023-10-09T04:41:46.528746+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 105 objects/s recovering 2023-10-09T04:41:47.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:47.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:47.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:47 smithi153 bash[59565]: cluster 2023-10-09T04:41:45.789700+0000 osd.5 (osd.5) 103 : cluster 0 7.0 scrub starts 2023-10-09T04:41:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:47 smithi153 bash[59565]: cluster 2023-10-09T04:41:46.425286+0000 osd.3 (osd.3) 3 : cluster 0 7.13 scrub starts 2023-10-09T04:41:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:47 smithi153 bash[59565]: cluster 2023-10-09T04:41:46.440283+0000 osd.3 (osd.3) 4 : cluster 0 7.13 scrub ok 2023-10-09T04:41:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:47 smithi153 bash[59565]: cluster 2023-10-09T04:41:46.528746+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v860: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 105 objects/s recovering 2023-10-09T04:41:47.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:47.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:41:47.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:47.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:48.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:48.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:48.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:48.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:48.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:48.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:48.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:48.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:48.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:48.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:48.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:48.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:48.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:48.878 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:48.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:48.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:49.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:49.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:49.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:49.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:49.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:49.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:49.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:49.499 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:49.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:49.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:49.634 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:49.646 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:49 smithi153 bash[59565]: cluster 2023-10-09T04:41:48.529628+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:49.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:49.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:49.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:49.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:49.912 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:49 smithi082 bash[79733]: cluster 2023-10-09T04:41:48.529628+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:49.912 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:49 smithi082 bash[77063]: cluster 2023-10-09T04:41:48.529628+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v861: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:49.912 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:49] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:41:49.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:49.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:49.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:50.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:50.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:50.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:50.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:50.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:50.225 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:50.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.323 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:50.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:50.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:50.545 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:50.562 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:50.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:50.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:50.718 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:50.721 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:50 smithi153 bash[59565]: cluster 2023-10-09T04:41:49.715652+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:50.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.831 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:50 smithi082 bash[79733]: cluster 2023-10-09T04:41:49.715652+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:50.832 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:50 smithi082 bash[77063]: cluster 2023-10-09T04:41:49.715652+0000 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:50.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:50.949 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:50.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:51.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:51.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:51.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:51.335 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:51.362 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:51.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:51.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:51.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.625 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:51.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:51.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:51.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:51.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.869 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:51 smithi153 bash[59565]: cluster 2023-10-09T04:41:50.530343+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 120 objects/s recovering 2023-10-09T04:41:51.870 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:51 smithi153 bash[59565]: cluster 2023-10-09T04:41:50.706201+0000 osd.5 (osd.5) 104 : cluster 0 7.0 scrub starts 2023-10-09T04:41:51.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:51.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:51 smithi082 bash[79733]: cluster 2023-10-09T04:41:50.530343+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 120 objects/s recovering 2023-10-09T04:41:51.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:51 smithi082 bash[79733]: cluster 2023-10-09T04:41:50.706201+0000 osd.5 (osd.5) 104 : cluster 0 7.0 scrub starts 2023-10-09T04:41:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:51 smithi082 bash[77063]: cluster 2023-10-09T04:41:50.530343+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v862: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 120 objects/s recovering 2023-10-09T04:41:51.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:51 smithi082 bash[77063]: cluster 2023-10-09T04:41:50.706201+0000 osd.5 (osd.5) 104 : cluster 0 7.0 scrub starts 2023-10-09T04:41:51.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:52.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:52.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:52.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:52.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.266 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:52.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:52.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:52.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:52.529 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:52.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:52.632 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:52.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:52.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:52.898 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:52.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:53.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:53.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:53.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:53.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:53.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:53.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:53.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:53.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:53.511 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:53.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:53.579 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:53.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:53.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:53.707 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:53.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:53.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:53.821 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:53 smithi153 bash[59565]: cluster 2023-10-09T04:41:52.531171+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 108 objects/s recovering 2023-10-09T04:41:53.822 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:53 smithi153 bash[59565]: cluster 2023-10-09T04:41:52.632956+0000 osd.5 (osd.5) 105 : cluster 0 7.0 scrub starts 2023-10-09T04:41:53.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:53.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:53.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:53.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:53.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:53 smithi082 bash[77063]: cluster 2023-10-09T04:41:52.531171+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 108 objects/s recovering 2023-10-09T04:41:53.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:53 smithi082 bash[77063]: cluster 2023-10-09T04:41:52.632956+0000 osd.5 (osd.5) 105 : cluster 0 7.0 scrub starts 2023-10-09T04:41:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:53 smithi082 bash[79733]: cluster 2023-10-09T04:41:52.531171+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v863: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3573/774 objects degraded (461.628%); 0 B/s, 108 objects/s recovering 2023-10-09T04:41:53.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:53 smithi082 bash[79733]: cluster 2023-10-09T04:41:52.632956+0000 osd.5 (osd.5) 105 : cluster 0 7.0 scrub starts 2023-10-09T04:41:54.066 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:54.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:54.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:54.220 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:54.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:54.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:54.446 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:54.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:54.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:54.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:54.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:54.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:54.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:54.821 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:54.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:54.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:54.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:54.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:54.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:54.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:54 smithi082 bash[79733]: cluster 2023-10-09T04:41:54.717041+0000 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:54.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:54 smithi082 bash[77063]: cluster 2023-10-09T04:41:54.717041+0000 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:55.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:55.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:54 smithi153 bash[59565]: cluster 2023-10-09T04:41:54.717041+0000 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 3573/774 objects degraded (461.628%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:55.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:55.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:55.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.353 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:41:55.353 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:41:55.354 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:41:55.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:55.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:55.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:55.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:55.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:55.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:55.872 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:55 smithi153 bash[59565]: cluster 2023-10-09T04:41:54.150672+0000 osd.6 (osd.6) 70 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:55.872 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:55 smithi153 bash[59565]: cluster 2023-10-09T04:41:54.532069+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 182 objects/s recovering 2023-10-09T04:41:55.872 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:55 smithi153 bash[59565]: cluster 2023-10-09T04:41:55.509184+0000 osd.5 (osd.5) 106 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:41:55.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:55.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:55.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:56.004 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[77063]: cluster 2023-10-09T04:41:54.150672+0000 osd.6 (osd.6) 70 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:56.004 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[77063]: cluster 2023-10-09T04:41:54.532069+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 182 objects/s recovering 2023-10-09T04:41:56.005 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[77063]: cluster 2023-10-09T04:41:55.509184+0000 osd.5 (osd.5) 106 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:41:56.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[79733]: cluster 2023-10-09T04:41:54.150672+0000 osd.6 (osd.6) 70 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:56.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[79733]: cluster 2023-10-09T04:41:54.532069+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v864: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 182 objects/s recovering 2023-10-09T04:41:56.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:55 smithi082 bash[79733]: cluster 2023-10-09T04:41:55.509184+0000 osd.5 (osd.5) 106 : cluster 0 7.0 deep-scrub starts 2023-10-09T04:41:56.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:56.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:56.126 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:56.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:56.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:56.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:56.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:56.409 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:56.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:56.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:56.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:56.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:56.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:56.789 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:56 smithi082 bash[77063]: cluster 2023-10-09T04:41:55.181997+0000 osd.6 (osd.6) 71 : cluster 0 7.7 scrub starts 2023-10-09T04:41:56.790 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:56 smithi082 bash[79733]: cluster 2023-10-09T04:41:55.181997+0000 osd.6 (osd.6) 71 : cluster 0 7.7 scrub starts 2023-10-09T04:41:56.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:56.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:56.939 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:56.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:56.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:57.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:57.052 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:56 smithi153 bash[59565]: cluster 2023-10-09T04:41:55.181997+0000 osd.6 (osd.6) 71 : cluster 0 7.7 scrub starts 2023-10-09T04:41:57.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:57.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:57.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:57.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:57.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:57.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:57.327 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:57.430 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:57.462 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:57.566 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:57.591 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:57.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:57.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:57.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:57.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:57.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:57.970 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.198092+0000 osd.6 (osd.6) 72 : cluster 0 7.7 scrub starts 2023-10-09T04:41:57.970 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.532970+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.737161+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.741841+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.742358+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.752085+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.752272+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.971 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:57 smithi153 bash[59565]: cluster 2023-10-09T04:41:56.752367+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.198092+0000 osd.6 (osd.6) 72 : cluster 0 7.7 scrub starts 2023-10-09T04:41:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.532970+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.737161+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.741841+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.742358+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.752085+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.752272+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[79733]: cluster 2023-10-09T04:41:56.752367+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.198092+0000 osd.6 (osd.6) 72 : cluster 0 7.7 scrub starts 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.532970+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v865: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3516/774 objects degraded (454.264%); 0 B/s, 129 objects/s recovering 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.737161+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.741841+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.742358+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:41:57.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.752085+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.752272+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:57 smithi082 bash[77063]: cluster 2023-10-09T04:41:56.752367+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T04:41:57.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:57.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:58.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:58.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:58.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:58.240 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:58.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:58.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:58.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:58.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:58.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:58.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:58.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:58.626 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:58.662 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:58.663 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:58.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:58.759 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:58.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:58.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.208634+0000 osd.6 (osd.6) 74 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:58.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.752015+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.874 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.756682+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.759943+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.762934+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.828275+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[79733]: cluster 2023-10-09T04:41:57.828390+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.208634+0000 osd.6 (osd.6) 74 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.752015+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.756682+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.759943+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:41:58.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.762934+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.876 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.828275+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.876 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:58 smithi082 bash[77063]: cluster 2023-10-09T04:41:57.828390+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:58.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:58.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:58.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.208634+0000 osd.6 (osd.6) 74 : cluster 0 7.7 deep-scrub starts 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.752015+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.756682+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.759943+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.762934+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.828275+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:59.003 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:58 smithi153 bash[59565]: cluster 2023-10-09T04:41:57.828390+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T04:41:59.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:59.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:59.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:59.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:59.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:59.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.415 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:59.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:41:59.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:59.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:59.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:59.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:59.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:59.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[79733]: cluster 2023-10-09T04:41:58.533694+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[79733]: audit 2023-10-09T04:41:59.281267+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[79733]: cluster 2023-10-09T04:41:59.511992+0000 osd.5 (osd.5) 109 : cluster 0 7.0 scrub starts 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[79733]: cluster 2023-10-09T04:41:59.718473+0000 mon.a (mon.0) 1230 : cluster 3 Health check update: Degraded data redundancy: 1224/774 objects degraded (158.140%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[77063]: cluster 2023-10-09T04:41:58.533694+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:41:59.969 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[77063]: audit 2023-10-09T04:41:59.281267+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:59.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[77063]: cluster 2023-10-09T04:41:59.511992+0000 osd.5 (osd.5) 109 : cluster 0 7.0 scrub starts 2023-10-09T04:41:59.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[77063]: cluster 2023-10-09T04:41:59.718473+0000 mon.a (mon.0) 1230 : cluster 3 Health check update: Degraded data redundancy: 1224/774 objects degraded (158.140%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:59.970 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:41:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:41:59] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:41:59.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:41:59.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:00.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:00.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:59 smithi153 bash[59565]: cluster 2023-10-09T04:41:58.533694+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v868: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:42:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:59 smithi153 bash[59565]: audit 2023-10-09T04:41:59.281267+0000 mon.a (mon.0) 1229 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:59 smithi153 bash[59565]: cluster 2023-10-09T04:41:59.511992+0000 osd.5 (osd.5) 109 : cluster 0 7.0 scrub starts 2023-10-09T04:42:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:41:59 smithi153 bash[59565]: cluster 2023-10-09T04:41:59.718473+0000 mon.a (mon.0) 1230 : cluster 3 Health check update: Degraded data redundancy: 1224/774 objects degraded (158.140%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:00.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:00.238 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:00.342 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:00.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:00.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:00.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:00.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:00.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:00.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:00.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:00.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:00.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:00.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:00.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:01.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:01.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:01.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:01.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:01.245 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:01.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:01.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:01.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:01.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:01.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:01.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:01.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:01.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:01.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:01.915 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:01.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:01.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:01.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:01 smithi082 bash[77063]: cluster 2023-10-09T04:42:00.534320+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:42:01.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:01 smithi082 bash[79733]: cluster 2023-10-09T04:42:00.534320+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:42:02.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:02.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:02.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:01 smithi153 bash[59565]: cluster 2023-10-09T04:42:00.534320+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v869: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 248 objects/s recovering 2023-10-09T04:42:02.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:02.195 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:42:02.195 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:42:02.195 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T04:42:02.196 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:42:02.196 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:42:02.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T04:42:02.207 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:02.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:02.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:02.400 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:42:02.436 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:42:02.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:02.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:02.475 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:42:02.514 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:42:02.551 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:42:02.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:02.578 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:42:02.604 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:42:02.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:02.627 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:42:02.628 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.628 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.629 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:42:02.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:02.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:02.801 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_2' removed 2023-10-09T04:42:02.814 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:42:02.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:02.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:02.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:02 smithi153 bash[59565]: audit 2023-10-09T04:42:02.354133+0000 mon.a (mon.0) 1231 : audit 0 from='client.? 172.21.15.82:0/1211865039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:02 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.486585+0000 osd.5 (osd.5) 110 : cluster 0 7.0 scrub starts 2023-10-09T04:42:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:02 smithi153 bash[59565]: audit 2023-10-09T04:42:02.624787+0000 mon.a (mon.0) 1232 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:42:02.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:02 smithi153 bash[59565]: audit 2023-10-09T04:42:02.677178+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:42:02.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[77063]: audit 2023-10-09T04:42:02.354133+0000 mon.a (mon.0) 1231 : audit 0 from='client.? 172.21.15.82:0/1211865039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.486585+0000 osd.5 (osd.5) 110 : cluster 0 7.0 scrub starts 2023-10-09T04:42:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[77063]: audit 2023-10-09T04:42:02.624787+0000 mon.a (mon.0) 1232 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:42:02.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[77063]: audit 2023-10-09T04:42:02.677178+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:42:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[79733]: audit 2023-10-09T04:42:02.354133+0000 mon.a (mon.0) 1231 : audit 0 from='client.? 172.21.15.82:0/1211865039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.486585+0000 osd.5 (osd.5) 110 : cluster 0 7.0 scrub starts 2023-10-09T04:42:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[79733]: audit 2023-10-09T04:42:02.624787+0000 mon.a (mon.0) 1232 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:42:02.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:02 smithi082 bash[79733]: audit 2023-10-09T04:42:02.677178+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:42:03.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:03.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:03.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:03.238 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.274 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:03.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:03.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:03.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:03.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:03.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:03.807 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:03.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:03.939 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:03.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:03.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.535113+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 136 objects/s recovering 2023-10-09T04:42:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.776120+0000 mon.a (mon.0) 1234 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.784139+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:03.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.786057+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: audit 2023-10-09T04:42:02.790347+0000 mon.a (mon.0) 1235 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: audit 2023-10-09T04:42:02.790461+0000 mon.a (mon.0) 1236 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[79733]: cluster 2023-10-09T04:42:02.790529+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.535113+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 136 objects/s recovering 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.776120+0000 mon.a (mon.0) 1234 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.784139+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.786057+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: audit 2023-10-09T04:42:02.790347+0000 mon.a (mon.0) 1235 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:42:03.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: audit 2023-10-09T04:42:02.790461+0000 mon.a (mon.0) 1236 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:42:03.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:03 smithi082 bash[77063]: cluster 2023-10-09T04:42:02.790529+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:42:04.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.535113+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v870: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 19 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 1224/774 objects degraded (158.140%); 0 B/s, 136 objects/s recovering 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.776120+0000 mon.a (mon.0) 1234 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.784139+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.786057+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: audit 2023-10-09T04:42:02.790347+0000 mon.a (mon.0) 1235 : audit 1 from='client.? 172.21.15.82:0/1345537231' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: audit 2023-10-09T04:42:02.790461+0000 mon.a (mon.0) 1236 : audit 1 from='client.? 172.21.15.82:0/273015368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:42:04.051 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:03 smithi153 bash[59565]: cluster 2023-10-09T04:42:02.790529+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:42:04.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:04.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:04.202 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:04.293 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:04.306 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:04.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:04.437 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:04.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:04.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:04.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:04.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:04.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:04.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:04.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.459139+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.802906+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.806775+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.827099+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.286653+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.459139+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.802906+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.806775+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.827099+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.840 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:04 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.286653+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:04.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:04.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:04.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:04.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:05.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:05.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:04 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.459139+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:04 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.802906+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:05.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:04 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.806775+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:42:05.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:04 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.827099+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:05.102 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:04 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.286653+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T04:42:05.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:05.126 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:42:05.126 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:42:05.127 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T04:42:05.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T04:42:05.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:05.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:05.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:05.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:05.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:05.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:05.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:05.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:05.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:05.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:05.794 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:05.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.951710+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:03.954069+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.535777+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v873: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.802439+0000 mon.a (mon.0) 1239 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.810175+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.908 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.812819+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[79733]: cluster 2023-10-09T04:42:04.814391+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.951710+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:03.954069+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.535777+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v873: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.802439+0000 mon.a (mon.0) 1239 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:05.909 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.810175+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.910 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.812819+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.910 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:05 smithi082 bash[77063]: cluster 2023-10-09T04:42:04.814391+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:42:05.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.951710+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:03.954069+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.535777+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v873: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.802439+0000 mon.a (mon.0) 1239 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.810175+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.812819+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T04:42:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:05 smithi153 bash[59565]: cluster 2023-10-09T04:42:04.814391+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:42:05.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:05.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:06.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:06.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:06.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:06.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:06.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:06.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:06.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:06.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:06.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:06.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:06.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:06.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:06.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:06.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:06.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:06 smithi082 bash[79733]: cluster 2023-10-09T04:42:06.805510+0000 mon.a (mon.0) 1241 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:42:06.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:06 smithi082 bash[77063]: cluster 2023-10-09T04:42:06.805510+0000 mon.a (mon.0) 1241 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:42:06.988 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:06 smithi153 bash[59565]: cluster 2023-10-09T04:42:06.805510+0000 mon.a (mon.0) 1241 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:42:07.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:07.080 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:42:07.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:07.140 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:07.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:07.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:07.287 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:07.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:07.422 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:07.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:07.525 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:07.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:07.655 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:07.683 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:07.786 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:07.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:42:07.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:07.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:07 smithi153 bash[59565]: cluster 2023-10-09T04:42:06.536636+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v875: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:07.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:07.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:08.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:07 smithi082 bash[77063]: cluster 2023-10-09T04:42:06.536636+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v875: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:08.074 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:07 smithi082 bash[79733]: cluster 2023-10-09T04:42:06.536636+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v875: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:08.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:08.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:08.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:08.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:08.475 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:08.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:08.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:08.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:08.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:08.834 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22573 6.e 6:793e1913:::benchmark_data_smithi082_128868_object22572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 589.80743949999999, 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23767322799999999, 2023-10-09T04:42:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22573 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020490+0000", 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020488+0000", 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7503999999999998e-05 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020536+0000", 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4269000000000001e-05 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020560+0000", 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1192999999999999e-05 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020571+0000", 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016770700000000001 2023-10-09T04:42:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020739+0000", 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000143914 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.020883+0000", 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016347700000000001 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.021046+0000", 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022837199999999999 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023330+0000", 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015380669999999999 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.024868+0000", 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23309624000000001 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.257964+0000", 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011879699999999999 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258083+0000", 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0285999999999997e-05 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258163+0000", 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23767322799999999 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22581 6.e 6:754d3d1d:::benchmark_data_smithi082_128868_object22580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 589.80519422099997, 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23624266899999999, 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22581 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022735+0000", 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022734+0000", 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3418999999999996e-05 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022808+0000", 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0966e-05 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022818+0000", 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1190000000000002e-06 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.022825+0000", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00054854399999999998 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023373+0000", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000153327 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023526+0000", 2023-10-09T04:42:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020472700000000001 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.023731+0000", 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001241736 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.024973+0000", 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00092022999999999996 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.025893+0000", 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.232957569 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258851+0000", 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0142999999999996e-05 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258921+0000", 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7206000000000001e-05 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.258978+0000", 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23624266899999999 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22589 6.e 6:78dafd59:::benchmark_data_smithi082_128868_object22588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 589.80179296899996, 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23663462099999999, 2023-10-09T04:42:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22589 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.852 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026137+0000", 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026135+0000", 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7635999999999997e-05 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026173+0000", 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8362000000000001e-05 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026191+0000", 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7409999999999998e-06 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026200+0000", 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6614999999999999e-05 2023-10-09T04:42:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026256+0000", 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000146713 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026403+0000", 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000186606 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.026590+0000", 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001099572 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.027689+0000", 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047519499999999998 2023-10-09T04:42:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.028164+0000", 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.234474934 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262639+0000", 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5122000000000003e-05 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262714+0000", 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6870000000000003e-05 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262771+0000", 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23663462099999999 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:22601 6.e 6:7da5b80c:::benchmark_data_smithi082_128868_object22600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 589.79314421499998, 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22826392000000001, 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 22601 2023-10-09T04:42:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034785+0000", 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034784+0000", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6398999999999998e-05 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034861+0000", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0020000000000006e-06 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034869+0000", 2023-10-09T04:42:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.569e-06 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.034872+0000", 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00035316999999999998 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035226+0000", 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000127016 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035353+0000", 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9709000000000004e-05 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.035452+0000", 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001076694 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.036529+0000", 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00034020799999999999 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.036869+0000", 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22608289600000001 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.262952+0000", 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4405000000000004e-05 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.263017+0000", 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2823000000000001e-05 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:19.263049+0000", 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22826392000000001 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25605 6.e 6:7b421b4b:::benchmark_data_smithi082_128868_object25604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 588.06369634099997, 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.150442087, 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25605 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764233+0000", 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764232+0000", 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9565999999999999e-05 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764282+0000", 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1229e-05 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764293+0000", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9510000000000003e-06 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764299+0000", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00051598900000000001 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764815+0000", 2023-10-09T04:42:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000123836 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.764939+0000", 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000120155 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.765059+0000", 2023-10-09T04:42:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034649339999999998 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.768524+0000", 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8259999999999999e-06 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.768533+0000", 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14597078099999999 2023-10-09T04:42:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914503+0000", 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010176800000000001 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914605+0000", 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0284999999999995e-05 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.914675+0000", 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.150442087 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25625 6.e 6:7b55b5f9:::benchmark_data_smithi082_128868_object25624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 588.05656585999998, 2023-10-09T04:42:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14644722900000001, 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25625 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771364+0000", 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771362+0000", 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4214000000000001e-05 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771407+0000", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8134e-05 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771425+0000", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8423999999999998e-05 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771443+0000", 2023-10-09T04:42:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016804999999999999 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771611+0000", 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014548400000000001 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771757+0000", 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018745599999999999 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.771944+0000", 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016346850000000001 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.773579+0000", 2023-10-09T04:42:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001057952 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.774637+0000", 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14293971699999999 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917577+0000", 2023-10-09T04:42:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000105895 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917682+0000", 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000128558 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917811+0000", 2023-10-09T04:42:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14644722900000001 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:25638 6.e 6:7908a266:::benchmark_data_smithi082_128868_object25637:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 588.04854219799995, 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13866558900000001, 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 25638 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:42:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779387+0000", 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779386+0000", 2023-10-09T04:42:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2234000000000002e-05 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779439+0000", 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2211e-05 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779451+0000", 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3349999999999997e-06 2023-10-09T04:42:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779456+0000", 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0542e-05 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779487+0000", 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1191000000000002e-05 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779578+0000", 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111891 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.779690+0000", 2023-10-09T04:42:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0012906339999999999 2023-10-09T04:42:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.780980+0000", 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00065067399999999996 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.781631+0000", 2023-10-09T04:42:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13629439099999999 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917925+0000", 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2866000000000003e-05 2023-10-09T04:42:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.917998+0000", 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4775000000000003e-05 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:20.918053+0000", 2023-10-09T04:42:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13866558900000001 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:26828 6.e 6:7f2ece57:::benchmark_data_smithi082_128868_object26827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 587.323832896, 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13063107199999999, 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 26828 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504097+0000", 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504095+0000", 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8353999999999999e-05 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504133+0000", 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4508000000000001e-05 2023-10-09T04:42:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504148+0000", 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0929999999999999e-06 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504154+0000", 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4948999999999999e-05 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504209+0000", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000126699 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504335+0000", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017422600000000001 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.504510+0000", 2023-10-09T04:42:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001165529 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.505675+0000", 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026620700000000002 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.505941+0000", 2023-10-09T04:42:08.897 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.12852864 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634470+0000", 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000151111 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634621+0000", 2023-10-09T04:42:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000106601 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:21.634728+0000", 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13063107199999999 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:31791 6.e 6:7488fad2:::benchmark_data_smithi082_128868_object31790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 584.43661021800006, 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11084435099999999, 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 31791 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391319+0000", 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391318+0000", 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3900000000000003e-05 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391362+0000", 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1357e-05 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391373+0000", 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:42:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391379+0000", 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0275000000000001e-05 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391440+0000", 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000163346 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391603+0000", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000213604 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391817+0000", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0019215269999999999 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393738+0000", 2023-10-09T04:42:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3773000000000003e-05 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393782+0000", 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10828449900000001 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502066+0000", 2023-10-09T04:42:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7272999999999999e-05 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502124+0000", 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.019e-05 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502164+0000", 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11084435099999999 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:31792 6.e 6:79f14372:::benchmark_data_smithi082_128868_object31791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 584.43642088000001, 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11087027300000001, 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 31792 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391509+0000", 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391508+0000", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3638999999999999e-05 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391561+0000", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1829999999999994e-06 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391569+0000", 2023-10-09T04:42:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.442e-06 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.391574+0000", 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001329513 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.392903+0000", 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2236000000000003e-05 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.392986+0000", 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112772 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393098+0000", 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00078891900000000001 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.393887+0000", 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00044481099999999998 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.394332+0000", 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.107957769 2023-10-09T04:42:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502290+0000", 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3843e-05 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502334+0000", 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5241999999999998e-05 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:32:24.502379+0000", 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11087027300000001 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:79283 6.7 6:efa62da7:::benchmark_data_smithi082_128868_object79282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T04:42:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 547.23883283099997, 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11057702599999999, 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 79283 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589097+0000", 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589095+0000", 2023-10-09T04:42:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7123e-05 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589173+0000", 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7784000000000001e-05 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589190+0000", 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3559999999999993e-06 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.589199+0000", 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00173497 2023-10-09T04:42:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.590934+0000", 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112447 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.591046+0000", 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018018299999999999 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3", 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.591226+0000", 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003313758 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.594540+0000", 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.10471435 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699254+0000", 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000150841 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699405+0000", 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026862300000000001 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.699674+0000", 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11057702599999999 2023-10-09T04:42:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:79289 6.6 6:6660557e:::benchmark_data_smithi082_128868_object79288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:42:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 547.23141973199995, 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114452888, 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 79289 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596510+0000", 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596508+0000", 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2818000000000006e-05 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596591+0000", 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4948e-05 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596606+0000", 2023-10-09T04:42:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4660000000000002e-06 2023-10-09T04:42:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596613+0000", 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3134999999999999e-05 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596666+0000", 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9699999999999998e-05 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596756+0000", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000101748 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.596858+0000", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027267839999999999 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.599584+0000", 2023-10-09T04:42:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000327559 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.599912+0000", 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.110935303 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710847+0000", 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7570999999999994e-05 2023-10-09T04:42:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710915+0000", 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7902999999999999e-05 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:33:01.710963+0000", 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.114452888 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98683 6.6 6:664c4778:::benchmark_data_smithi082_128868_object11345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 465.96711839199997, 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23657092900000001, 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98683 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860811+0000", 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860810+0000", 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2110000000000001e-06 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860817+0000", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.38e-05 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860831+0000", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2699999999999999e-06 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860838+0000", 2023-10-09T04:42:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0358999999999998e-05 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860878+0000", 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010762100000000001 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860986+0000", 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5366999999999998e-05 2023-10-09T04:42:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.861071+0000", 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0053058259999999996 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.866377+0000", 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049759499999999998 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.871353+0000", 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22573111000000001 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097084+0000", 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00023964599999999999 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097324+0000", 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8329999999999999e-05 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097382+0000", 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23657092900000001 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.34995.0:98684 6.6 6:69085b6d:::benchmark_data_smithi082_128868_object11346:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 465.96698227399997, 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23656778000000001, 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.34995", 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/2805339738", 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 98684 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860947+0000", 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860946+0000", 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8239999999999996e-06 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860952+0000", 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4110000000000002e-06 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860962+0000", 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6619999999999999e-06 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.860968+0000", 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026980799999999998 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863666+0000", 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012220800000000001 2023-10-09T04:42:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863789+0000", 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014059200000000001 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.863929+0000", 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002663117 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.866592+0000", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049010529999999998 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:22.871493+0000", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22593812799999999 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097431+0000", 2023-10-09T04:42:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4642999999999999e-05 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097476+0000", 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9042000000000003e-05 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:34:23.097515+0000", 2023-10-09T04:42:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.23656778000000001 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:18099 7.1 7:85c7f05b:::benchmark_data_smithi082_155217_object18098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:37:44.344080+0000", 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 264.483849437, 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.096239018999999995, 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 18099 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344080+0000", 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.949 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344080+0000", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344078+0000", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5318000000000002e-05 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344124+0000", 2023-10-09T04:42:08.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5968e-05 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344140+0000", 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8560000000000002e-06 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.951 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344149+0000", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4790000000000006e-05 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344234+0000", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013892 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344373+0000", 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025412699999999999 2023-10-09T04:42:08.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.344627+0000", 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001133015 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.345760+0000", 2023-10-09T04:42:08.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00036525700000000002 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.346126+0000", 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.093999841000000001 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.440125+0000", 2023-10-09T04:42:08.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9884999999999997e-05 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.440225+0000", 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3908999999999998e-05 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:44.440319+0000", 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.096239018999999995 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.955 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:27374 7.1 7:8380cb46:::benchmark_data_smithi082_155217_object27373:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:37:49.571193+0000", 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 259.25673677100002, 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074208695000000005, 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.956 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 27374 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571193+0000", 2023-10-09T04:42:08.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571193+0000", 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571191+0000", 2023-10-09T04:42:08.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6421000000000001e-05 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571248+0000", 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2989e-05 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.959 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571261+0000", 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5979999999999996e-06 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571266+0000", 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0652000000000002e-05 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571317+0000", 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015238599999999999 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571469+0000", 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013058000000000001 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:42:08.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.571600+0000", 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0012604820000000001 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.572860+0000", 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000434115 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.573295+0000", 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.071930364999999996 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.645225+0000", 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8554000000000005e-05 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.645323+0000", 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8094000000000001e-05 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:37:49.645402+0000", 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074208695000000005 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:89248 7.1 7:8f1e01e8:::benchmark_data_smithi082_155217_object89247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:38:45.210969+0000", 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 203.616960949, 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.056007784999999997, 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.965 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 89248 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.210969+0000", 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.966 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.210969+0000", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.210967+0000", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4307999999999995e-05 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.211031+0000", 2023-10-09T04:42:08.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7564e-05 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.211048+0000", 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5550000000000003e-06 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.211057+0000", 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049504869999999999 2023-10-09T04:42:08.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.216007+0000", 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000182311 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.216190+0000", 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000172702 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.216362+0000", 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0070782060000000001 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.223441+0000", 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002832883 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.226274+0000", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.040594684999999998 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.266868+0000", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2540999999999996e-05 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.266931+0000", 2023-10-09T04:42:08.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5692000000000002e-05 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.266976+0000", 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.056007784999999997 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:89261 7.1 7:8de011f7:::benchmark_data_smithi082_155217_object89260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:38:45.225097+0000", 2023-10-09T04:42:08.972 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 203.60283287799999, 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.049926964999999997, 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 89261 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225097+0000", 2023-10-09T04:42:08.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225097+0000", 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225095+0000", 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2441000000000007e-05 2023-10-09T04:42:08.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225167+0000", 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6854000000000002e-05 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225184+0000", 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2679999999999994e-06 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.225192+0000", 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001103166 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.226296+0000", 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012946400000000001 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.226425+0000", 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014908900000000001 2023-10-09T04:42:08.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.226574+0000", 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023242050000000002 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.228898+0000", 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5527000000000001e-05 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.977 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.228934+0000", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.046029319999999999 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.274963+0000", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4876000000000003e-05 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.274998+0000", 2023-10-09T04:42:08.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5673e-05 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:38:45.275024+0000", 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.049926964999999997 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.979 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.35259.0:195200 7.8 e209/208 7:12c5bf0f:::benchmark_data_smithi082_155217_object92540:head v 209'12691, mlcod=207'12681)", 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:41:27.656037+0000", 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 41.171892546999999, 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.056908321999999997, 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656037+0000", 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656037+0000", 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656036+0000", 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0655999999999999e-05 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656047+0000", 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9644999999999997e-05 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656086+0000", 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9529999999999999e-06 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.656094+0000", 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.020227261999999999 2023-10-09T04:42:08.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.676321+0000", 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1705999999999998e-05 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.676353+0000", 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.036550527999999999 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.712904+0000", 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1771999999999999e-05 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.712945+0000", 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.056908321999999997 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.35259.0:195232 7.8 e209/208 7:15e737ae:::benchmark_data_smithi082_155217_object92572:head v 209'12692, mlcod=207'12681)", 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:41:27.660104+0000", 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 41.167825469999997, 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.052862333999999997, 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:08.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660104+0000", 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660104+0000", 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660103+0000", 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8740000000000007e-06 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660113+0000", 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5264000000000002e-05 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:08.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660128+0000", 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4470000000000003e-06 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.660135+0000", 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.021071400000000001 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.681206+0000", 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0307e-05 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.681236+0000", 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.031713147999999997 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:42:08.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.712950+0000", 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6986999999999998e-05 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:41:27.712966+0000", 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.052862333999999997 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.990 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.991 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.991 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:08.991 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:08.991 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:42:08.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:09.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:09.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:09.153 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:09.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:09.283 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:09.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:09.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:09.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:09.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:09.658 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:09.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:09.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:09.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:09.890 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:09.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:09.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:09.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:10.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:10.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:10.093 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:09 smithi082 bash[77063]: cluster 2023-10-09T04:42:08.537528+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v876: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:10.093 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:09] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-09T04:42:10.124 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:09 smithi153 bash[59565]: cluster 2023-10-09T04:42:08.537528+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v876: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:10.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:10.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:10.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:10.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:10.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:10.449 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:10 smithi082 bash[79733]: cluster 2023-10-09T04:42:08.537528+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v876: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:10.471 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:10.550 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:42:10.575 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:10.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:10.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:10.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:10.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:10.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:10.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:10.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:10.909 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:10.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:11.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:11.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:11.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:11.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:11.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:11.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:11.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:11.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:11.922 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:11.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:11.942 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:11 smithi153 bash[59565]: cluster 2023-10-09T04:42:10.538213+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v877: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:11.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:11.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:11.967 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:42:11.967 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:42:11.967 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:42:12.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:12.095 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:12.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:12.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:12.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:12.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:12.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:11 smithi082 bash[77063]: cluster 2023-10-09T04:42:10.538213+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v877: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:12.276 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:11 smithi082 bash[79733]: cluster 2023-10-09T04:42:10.538213+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v877: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:12.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:12.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:12.454 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:12.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:12.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:12.606 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:12.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:12.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:12.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:12.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:12.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:12.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:13.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:13.142 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:13.246 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.277 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:13.381 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:13.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:13.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:13.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:13.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:13.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:13.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:13.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:13.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:13.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:14.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:14.055 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:13 smithi153 bash[59565]: cluster 2023-10-09T04:42:12.538980+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v878: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:14.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:14.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:14.190 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:13 smithi082 bash[77063]: cluster 2023-10-09T04:42:12.538980+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v878: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:14.191 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:13 smithi082 bash[79733]: cluster 2023-10-09T04:42:12.538980+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v878: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:42:14.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:14.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:14.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:14.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:14.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:14.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:14.591 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:14.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:14.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:14.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:14.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:14.860 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:14.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:14.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:14.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:15.098 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:15.127 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:15.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:15.262 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:15.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:15.370 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:15.391 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[77063]: audit 2023-10-09T04:42:14.285928+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.391 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[77063]: audit 2023-10-09T04:42:14.287178+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:15.391 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[77063]: cluster 2023-10-09T04:42:14.539561+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v879: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:15.392 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[77063]: audit 2023-10-09T04:42:14.792896+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[79733]: audit 2023-10-09T04:42:14.285928+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[79733]: audit 2023-10-09T04:42:14.287178+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:15.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[79733]: cluster 2023-10-09T04:42:14.539561+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v879: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:15.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:15 smithi082 bash[79733]: audit 2023-10-09T04:42:14.792896+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:15.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:15.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:15.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:15.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:15.653 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:15 smithi153 bash[59565]: audit 2023-10-09T04:42:14.285928+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:15 smithi153 bash[59565]: audit 2023-10-09T04:42:14.287178+0000 mon.a (mon.0) 1243 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:15 smithi153 bash[59565]: cluster 2023-10-09T04:42:14.539561+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v879: 73 pgs: 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:15.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:15 smithi153 bash[59565]: audit 2023-10-09T04:42:14.792896+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:15.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:15.788 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:15.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:15.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:15.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:16.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:16.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:16.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:16.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:16.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:16.315 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:16.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:16.448 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:16.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:16.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:16.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:16.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:16.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:16.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:16.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:16.977 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:16.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:17.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:17.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:17.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:17.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:17.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:17.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:17.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:17.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:17.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:17.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:17.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:17 smithi153 bash[59565]: cluster 2023-10-09T04:42:16.540479+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:17.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:17.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:17.854 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:42:17.854 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:42:17.854 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-09T04:42:17.854 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T04:42:17.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T04:42:17.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:17.862 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:17 smithi082 bash[77063]: cluster 2023-10-09T04:42:16.540479+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:17.863 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:17 smithi082 bash[79733]: cluster 2023-10-09T04:42:16.540479+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:42:17.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:17.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:17.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:18.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:18.151 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:17 smithi082 systemd[1]: Stopping Ceph osd.2 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:42:18.252 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:18.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:18.386 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:18.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:18.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:18.643 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:42:18.644 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:42:18.644 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T04:42:18.644 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:42:18.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:18.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:18.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:18.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:19.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.054 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:18 smithi082 bash[91125]: debug 2023-10-09T04:42:18.848+0000 7fb398b0a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:42:19.054 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:18 smithi082 bash[91125]: debug 2023-10-09T04:42:18.848+0000 7fb398b0a700 -1 osd.2 217 *** Got signal Terminated *** 2023-10-09T04:42:19.054 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:18 smithi082 bash[91125]: debug 2023-10-09T04:42:18.848+0000 7fb398b0a700 -1 osd.2 217 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:42:19.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:19.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:19.207 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:19.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:19.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:19.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:42:19.460 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:19.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:19.585 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:19.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:19.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:19.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:19.927 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_3' created 2023-10-09T04:42:19.937 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:19] "GET /metrics HTTP/1.1" 200 34120 "" "Prometheus/2.33.4" 2023-10-09T04:42:19.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:19.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:19.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:20.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:20.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:20.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:19 smithi153 bash[59565]: cluster 2023-10-09T04:42:18.541349+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:20.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:19 smithi153 bash[59565]: cluster 2023-10-09T04:42:18.853924+0000 mon.a (mon.0) 1245 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:42:20.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:19 smithi082 bash[77063]: cluster 2023-10-09T04:42:18.541349+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:19 smithi082 bash[77063]: cluster 2023-10-09T04:42:18.853924+0000 mon.a (mon.0) 1245 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:42:20.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:19 smithi082 bash[79733]: cluster 2023-10-09T04:42:18.541349+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:20.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:19 smithi082 bash[79733]: cluster 2023-10-09T04:42:18.853924+0000 mon.a (mon.0) 1245 : cluster 1 osd.2 marked itself down and dead 2023-10-09T04:42:20.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:20.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:20.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:20.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:20.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:20.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:20.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:20.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:20.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:20.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:20.878 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:20.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:20.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:20.981 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:21.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:21.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:21.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:21.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: audit 2023-10-09T04:42:19.443161+0000 mon.a (mon.0) 1246 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:42:21.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.892100+0000 mon.a (mon.0) 1247 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:42:21.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.902787+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: audit 2023-10-09T04:42:19.903492+0000 mon.a (mon.0) 1248 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.903543+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.904798+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.905886+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.908375+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.909258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:19.909978+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: audit 2023-10-09T04:42:19.443161+0000 mon.a (mon.0) 1246 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.892100+0000 mon.a (mon.0) 1247 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:42:21.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.902787+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: audit 2023-10-09T04:42:19.903492+0000 mon.a (mon.0) 1248 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.903543+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.904798+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.905886+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.908375+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.909258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:19.909978+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: audit 2023-10-09T04:42:19.443161+0000 mon.a (mon.0) 1246 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.892100+0000 mon.a (mon.0) 1247 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.902787+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: audit 2023-10-09T04:42:19.903492+0000 mon.a (mon.0) 1248 : audit 1 from='client.? 172.21.15.82:0/3724849759' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.903543+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.904798+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.905886+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.908375+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.909258+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:20 smithi153 bash[59565]: cluster 2023-10-09T04:42:19.909978+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T04:42:21.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:21.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:21.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:21.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:21.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:21.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:21.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:21.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:21.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:21.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:21.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:21.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:21.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:22.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.542198+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v883: 89 pgs: 5 peering, 6 creating+peering, 10 unknown, 4 stale+active+clean, 64 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.900940+0000 mon.a (mon.0) 1250 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.910249+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.911220+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.911323+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.913696+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.913770+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.913996+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.049 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.914796+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.915770+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.916910+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.917680+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.917808+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.920077+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[77063]: cluster 2023-10-09T04:42:20.920755+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.542198+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v883: 89 pgs: 5 peering, 6 creating+peering, 10 unknown, 4 stale+active+clean, 64 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.900940+0000 mon.a (mon.0) 1250 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.910249+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.911220+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.050 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.911323+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.913696+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.913770+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.913996+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.914796+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.915770+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.916910+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.917680+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.917808+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.920077+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.051 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:21 smithi082 bash[79733]: cluster 2023-10-09T04:42:20.920755+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:22.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.542198+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v883: 89 pgs: 5 peering, 6 creating+peering, 10 unknown, 4 stale+active+clean, 64 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.900940+0000 mon.a (mon.0) 1250 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.910249+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.911220+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.911323+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.913696+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.913770+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.913996+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.914796+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.915770+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.916910+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.917680+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.917808+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.920077+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:21 smithi153 bash[59565]: cluster 2023-10-09T04:42:20.920755+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T04:42:22.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:22.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:22.328 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:22.432 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:22.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:22.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:22.697 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:22.718 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:22.821 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:22.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:22.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[77063]: cluster 2023-10-09T04:42:21.910586+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[77063]: cluster 2023-10-09T04:42:21.911239+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[77063]: cluster 2023-10-09T04:42:21.913201+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:42:22.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[77063]: cluster 2023-10-09T04:42:21.914784+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[77063]: cluster 2023-10-09T04:42:21.914819+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[79733]: cluster 2023-10-09T04:42:21.910586+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[79733]: cluster 2023-10-09T04:42:21.911239+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[79733]: cluster 2023-10-09T04:42:21.913201+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[79733]: cluster 2023-10-09T04:42:21.914784+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:22 smithi082 bash[79733]: cluster 2023-10-09T04:42:21.914819+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:22.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:22.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:23.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:23.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:22 smithi153 bash[59565]: cluster 2023-10-09T04:42:21.910586+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:22 smithi153 bash[59565]: cluster 2023-10-09T04:42:21.911239+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:22 smithi153 bash[59565]: cluster 2023-10-09T04:42:21.913201+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:42:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:22 smithi153 bash[59565]: cluster 2023-10-09T04:42:21.914784+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:23.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:22 smithi153 bash[59565]: cluster 2023-10-09T04:42:21.914819+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T04:42:23.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:23.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:23.350 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:23.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:23.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:23.420 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:42:23.445 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:42:23.456 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[173616]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T04:42:23.470 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:42:23.482 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T04:42:23.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:23.506 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:42:23.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:23.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:23.531 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:42:23.557 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:42:23.583 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.584 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:42:23.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:23.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:23.731 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[174098]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T04:42:23.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:23.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:23.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:23.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:23.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:23.932 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:42:24.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:24.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:24.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:24.135 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[77063]: cluster 2023-10-09T04:42:22.542807+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v886: 89 pgs: 7 peering, 6 creating+peering, 10 unknown, 3 stale+active+clean, 63 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:24.135 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[77063]: audit 2023-10-09T04:42:23.397173+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.82:0/2789936190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:24.135 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[77063]: audit 2023-10-09T04:42:23.893401+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.82:0/4285873133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.135 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[77063]: audit 2023-10-09T04:42:23.893840+0000 mon.a (mon.0) 1253 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[79733]: cluster 2023-10-09T04:42:22.542807+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v886: 89 pgs: 7 peering, 6 creating+peering, 10 unknown, 3 stale+active+clean, 63 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:24.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[79733]: audit 2023-10-09T04:42:23.397173+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.82:0/2789936190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:24.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[79733]: audit 2023-10-09T04:42:23.893401+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.82:0/4285873133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:23 smithi082 bash[79733]: audit 2023-10-09T04:42:23.893840+0000 mon.a (mon.0) 1253 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:23 smithi153 bash[59565]: cluster 2023-10-09T04:42:22.542807+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v886: 89 pgs: 7 peering, 6 creating+peering, 10 unknown, 3 stale+active+clean, 63 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:42:24.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:23 smithi153 bash[59565]: audit 2023-10-09T04:42:23.397173+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.82:0/2789936190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:24.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:23 smithi153 bash[59565]: audit 2023-10-09T04:42:23.893401+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.82:0/4285873133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.195 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:23 smithi153 bash[59565]: audit 2023-10-09T04:42:23.893840+0000 mon.a (mon.0) 1253 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:24.233 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:24.297 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:24.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:24.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:24.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:24.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:24.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:24.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:24.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:24.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:24.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:25.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: cluster 2023-10-09T04:42:23.918538+0000 mon.a (mon.0) 1254 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: cluster 2023-10-09T04:42:23.921713+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: cluster 2023-10-09T04:42:23.923948+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: audit 2023-10-09T04:42:23.926379+0000 mon.a (mon.0) 1255 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:25.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: cluster 2023-10-09T04:42:23.926465+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e221: 8 total, 7 up, 8 in 2023-10-09T04:42:25.101 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:24 smithi153 bash[59565]: cluster 2023-10-09T04:42:24.683005+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:25.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: cluster 2023-10-09T04:42:23.918538+0000 mon.a (mon.0) 1254 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: cluster 2023-10-09T04:42:23.921713+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: cluster 2023-10-09T04:42:23.923948+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: audit 2023-10-09T04:42:23.926379+0000 mon.a (mon.0) 1255 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: cluster 2023-10-09T04:42:23.926465+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e221: 8 total, 7 up, 8 in 2023-10-09T04:42:25.243 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[79733]: cluster 2023-10-09T04:42:24.683005+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: cluster 2023-10-09T04:42:23.918538+0000 mon.a (mon.0) 1254 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: cluster 2023-10-09T04:42:23.921713+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: cluster 2023-10-09T04:42:23.923948+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: audit 2023-10-09T04:42:23.926379+0000 mon.a (mon.0) 1255 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: cluster 2023-10-09T04:42:23.926465+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e221: 8 total, 7 up, 8 in 2023-10-09T04:42:25.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:24 smithi082 bash[77063]: cluster 2023-10-09T04:42:24.683005+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:25.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.383 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:25.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:25.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:25.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:25.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:25.748 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:25.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:25.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:25.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:25.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[77063]: cluster 2023-10-09T04:42:24.543564+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v888: 89 pgs: 1 active+undersized+degraded, 17 active+undersized, 7 peering, 6 creating+peering, 58 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:42:25.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[77063]: cluster 2023-10-09T04:42:24.683031+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.938 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[77063]: cluster 2023-10-09T04:42:24.928467+0000 mon.a (mon.0) 1257 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:42:25.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:25 smithi153 bash[59565]: cluster 2023-10-09T04:42:24.543564+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v888: 89 pgs: 1 active+undersized+degraded, 17 active+undersized, 7 peering, 6 creating+peering, 58 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:42:25.975 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:25 smithi153 bash[59565]: cluster 2023-10-09T04:42:24.683031+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:25.975 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:25 smithi153 bash[59565]: cluster 2023-10-09T04:42:24.928467+0000 mon.a (mon.0) 1257 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:42:26.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.049 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:26.126 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[79733]: cluster 2023-10-09T04:42:24.543564+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v888: 89 pgs: 1 active+undersized+degraded, 17 active+undersized, 7 peering, 6 creating+peering, 58 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T04:42:26.126 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[79733]: cluster 2023-10-09T04:42:24.683031+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T04:42:26.126 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:25 smithi082 bash[79733]: cluster 2023-10-09T04:42:24.928467+0000 mon.a (mon.0) 1257 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:42:26.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:26.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:26.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:26.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:26.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:26.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:26.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:26.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:26.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:26.685 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:42:26.731 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:42:26.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.820 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:26.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:26.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:26.949 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:26.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:26.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:26 smithi082 bash[77063]: audit 2023-10-09T04:42:26.264927+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:26.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:26 smithi082 bash[79733]: audit 2023-10-09T04:42:26.264927+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:27.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:27.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:26 smithi153 bash[59565]: audit 2023-10-09T04:42:26.264927+0000 mon.a (mon.0) 1258 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:42:27.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:27.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:27.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:27.318 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:27.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:27.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:27.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:27.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:27.731 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:27.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:27.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:27.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:27.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:28.007 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:27 smithi153 bash[59565]: cluster 2023-10-09T04:42:26.544167+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v889: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:28.007 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:27 smithi153 bash[59565]: cluster 2023-10-09T04:42:26.935588+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T04:42:28.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:28.081 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:42:28.081 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:42:28.131 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:28.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:27 smithi082 bash[77063]: cluster 2023-10-09T04:42:26.544167+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v889: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:28.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:27 smithi082 bash[77063]: cluster 2023-10-09T04:42:26.935588+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T04:42:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:27 smithi082 bash[79733]: cluster 2023-10-09T04:42:26.544167+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v889: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:27 smithi082 bash[79733]: cluster 2023-10-09T04:42:26.935588+0000 mon.a (mon.0) 1259 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T04:42:28.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:28.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.418 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:28.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:28.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:28.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:28.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:28.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:42:28.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T04:42:28.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:28.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:28.830 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:42:28.830 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T04:42:28.906 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:42:28.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:42:28.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:28.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:29.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:29.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:29.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:29.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:29.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:29.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:29.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:29.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:29] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-09T04:42:29.731 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:29.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:29.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:29.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:29.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:29.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:30.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:30.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:30.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:29 smithi153 bash[59565]: cluster 2023-10-09T04:42:28.545053+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v890: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:30.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:29 smithi153 bash[59565]: audit 2023-10-09T04:42:29.288765+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:30.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:29 smithi153 bash[59565]: audit 2023-10-09T04:42:29.290412+0000 mon.a (mon.0) 1261 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:30.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:30.212 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[77063]: cluster 2023-10-09T04:42:28.545053+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v890: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[77063]: audit 2023-10-09T04:42:29.288765+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[77063]: audit 2023-10-09T04:42:29.290412+0000 mon.a (mon.0) 1261 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[79733]: cluster 2023-10-09T04:42:28.545053+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v890: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[79733]: audit 2023-10-09T04:42:29.288765+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:29 smithi082 bash[79733]: audit 2023-10-09T04:42:29.290412+0000 mon.a (mon.0) 1261 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:30.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:30.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:30.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:30.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:30.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:30.534 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:30.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:30.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:30.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:30.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:30.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:30.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:30.949 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:30.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:30.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:31.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:31.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:31.217 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.247 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:31.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:31.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:31.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:31.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:31.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:31.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:31.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.791 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:31.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:31.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:31.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:31.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:32.030 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:32.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.175 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:31 smithi082 bash[77063]: cluster 2023-10-09T04:42:30.546243+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v891: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:32.175 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:31 smithi082 bash[79733]: cluster 2023-10-09T04:42:30.546243+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v891: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:32.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:31 smithi153 bash[59565]: cluster 2023-10-09T04:42:30.546243+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v891: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:32.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:32.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:32.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:32.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:32.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:32.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.714 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:32.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:32.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:32.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:32.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:32.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:33.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:33.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:33.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:33.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:33.256 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:33.256 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:33.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:33.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:33.453 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:33.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:33.479 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:32.360252+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.479 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:32.368660+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.479 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: cluster 2023-10-09T04:42:32.546876+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v892: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:33.479 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:32.874912+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.480 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:32.883596+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.480 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:33.352104+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:33.480 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:33 smithi153 bash[59565]: audit 2023-10-09T04:42:33.353554+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:33.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:33.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:33.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:33.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:32.360252+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:32.368660+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: cluster 2023-10-09T04:42:32.546876+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v892: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:32.874912+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:32.883596+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:33.352104+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[77063]: audit 2023-10-09T04:42:33.353554+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:32.360252+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:32.368660+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: cluster 2023-10-09T04:42:32.546876+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v892: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:33.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:32.874912+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:32.883596+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:33.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:33.352104+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:33.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:33 smithi082 bash[79733]: audit 2023-10-09T04:42:33.353554+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:33.733 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:33.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:33.832 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 7, 6, 5] 2023-10-09T04:42:33.832 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:42:33.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:33.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:33.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:33.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:34.023 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:34.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:34.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:34.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:34.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.399 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:34 smithi153 bash[59565]: audit 2023-10-09T04:42:33.371841+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:34.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:34.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:34.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:34.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:34.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:34.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:34 smithi082 bash[77063]: audit 2023-10-09T04:42:33.371841+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:34.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:34 smithi082 bash[79733]: audit 2023-10-09T04:42:33.371841+0000 mon.a (mon.0) 1268 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:34.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:34.829 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:34.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:34.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:35.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:35.202 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:35.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:35.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:35.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:35.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:35.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:35.487 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:35.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:35.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:35.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:35.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:35.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:35.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:35.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:36.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:36.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:35 smithi153 bash[59565]: cluster 2023-10-09T04:42:34.547911+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v893: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:36.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:35 smithi153 bash[59565]: audit 2023-10-09T04:42:34.811224+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:36.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:35 smithi153 bash[59565]: cluster 2023-10-09T04:42:34.896610+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:36.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:35 smithi153 bash[59565]: audit 2023-10-09T04:42:35.618605+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.82:0/1988150748' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.038 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:35 smithi153 bash[59565]: audit 2023-10-09T04:42:35.619061+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[79733]: cluster 2023-10-09T04:42:34.547911+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v893: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[79733]: audit 2023-10-09T04:42:34.811224+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[79733]: cluster 2023-10-09T04:42:34.896610+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[79733]: audit 2023-10-09T04:42:35.618605+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.82:0/1988150748' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[79733]: audit 2023-10-09T04:42:35.619061+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[77063]: cluster 2023-10-09T04:42:34.547911+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v893: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[77063]: audit 2023-10-09T04:42:34.811224+0000 mon.a (mon.0) 1269 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:42:36.057 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[77063]: cluster 2023-10-09T04:42:34.896610+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:36.058 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[77063]: audit 2023-10-09T04:42:35.618605+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.82:0/1988150748' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.058 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:35 smithi082 bash[77063]: audit 2023-10-09T04:42:35.619061+0000 mon.a (mon.0) 1271 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:42:36.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:36.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:36.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:36.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:36.373 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:36.388 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T04:42:36.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:36.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:36.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:36.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:36.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:36.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:36.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:36.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:36.927 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:36.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:37.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.058 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:37.133 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:42:37.133 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:42:37.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:37.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:37.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:37.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:37.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:37 smithi153 bash[59565]: cluster 2023-10-09T04:42:36.379074+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:37 smithi153 bash[59565]: cluster 2023-10-09T04:42:36.379147+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:37 smithi153 bash[59565]: audit 2023-10-09T04:42:36.383489+0000 mon.a (mon.0) 1272 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:37 smithi153 bash[59565]: cluster 2023-10-09T04:42:36.383689+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e222: 8 total, 7 up, 8 in 2023-10-09T04:42:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:37 smithi153 bash[59565]: cluster 2023-10-09T04:42:36.548584+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v895: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:37.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:37.466 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[77063]: cluster 2023-10-09T04:42:36.379074+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.466 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[77063]: cluster 2023-10-09T04:42:36.379147+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[77063]: audit 2023-10-09T04:42:36.383489+0000 mon.a (mon.0) 1272 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[77063]: cluster 2023-10-09T04:42:36.383689+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e222: 8 total, 7 up, 8 in 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[77063]: cluster 2023-10-09T04:42:36.548584+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v895: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[79733]: cluster 2023-10-09T04:42:36.379074+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[79733]: cluster 2023-10-09T04:42:36.379147+0000 osd.4 (osd.4) 170 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[79733]: audit 2023-10-09T04:42:36.383489+0000 mon.a (mon.0) 1272 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[79733]: cluster 2023-10-09T04:42:36.383689+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e222: 8 total, 7 up, 8 in 2023-10-09T04:42:37.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:37 smithi082 bash[79733]: cluster 2023-10-09T04:42:36.548584+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v895: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:37.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:37.576 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:37.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:37.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:37.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:37.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:37.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:37.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:38.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:42:38.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:38.119 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:38.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:38.246 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:38.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:38.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:38.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:38.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:38 smithi082 bash[77063]: cluster 2023-10-09T04:42:37.158364+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:38.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:38 smithi082 bash[79733]: cluster 2023-10-09T04:42:37.158364+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:38.506 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:38.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:38.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:38.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:38 smithi153 bash[59565]: cluster 2023-10-09T04:42:37.158364+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T04:42:38.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:38.772 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:38.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:38.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:38.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:39.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:39.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:39.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:39.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:39.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:39.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:39 smithi153 bash[59565]: cluster 2023-10-09T04:42:38.549573+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v896: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:39.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:39.462 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T04:42:39.462 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-10-09T04:42:39.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:42:39.487 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:42:39.487 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:42:39.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.553 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:39 smithi082 bash[77063]: cluster 2023-10-09T04:42:38.549573+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v896: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:39.554 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:39 smithi082 bash[79733]: cluster 2023-10-09T04:42:38.549573+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v896: 89 pgs: 3 active+undersized+degraded, 24 active+undersized, 62 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T04:42:39.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:39.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:39.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:39.709 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:42:39.710 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:42:39.710 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_175677 2023-10-09T04:42:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:42:39.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:42:39.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:39.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:39.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:39.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:39.943 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:39.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:39.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:39] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-09T04:42:40.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:40.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:40.341 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:40.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:40.452 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:42:40.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:40.490 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:42:40.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:40.522 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:42:40.558 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:42:40.593 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:42:40.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:40.630 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:42:40.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:40.668 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:42:40.668 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.669 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:42:40.710 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 16 2870 2854 178.365 178.375 0.00788851 0.00557076 2023-10-09T04:42:40.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:40.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:40.865 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:40.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:40.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:40.925 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:42:40.998 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:41.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:41.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:41.126 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:41.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:40 smithi153 bash[59565]: audit 2023-10-09T04:42:40.411048+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.82:0/3721447950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:41.135 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:40 smithi153 bash[59565]: audit 2023-10-09T04:42:40.729208+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:41.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:41.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:41.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:40 smithi082 bash[77063]: audit 2023-10-09T04:42:40.411048+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.82:0/3721447950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:41.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:40 smithi082 bash[77063]: audit 2023-10-09T04:42:40.729208+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:41.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:40 smithi082 bash[79733]: audit 2023-10-09T04:42:40.411048+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.82:0/3721447950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:41.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:40 smithi082 bash[79733]: audit 2023-10-09T04:42:40.729208+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:41.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:41.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:41.389 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.417 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:41.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:41.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:41.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:41.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:41.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 15 6076 6061 189.393 200.438 0.00400718 0.00526802 2023-10-09T04:42:41.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:41.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:41.930 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:41.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:42.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.550352+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v897: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.900034+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 153/1626 objects degraded (9.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.900568+0000 mon.a (mon.0) 1276 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.914299+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.915643+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: audit 2023-10-09T04:42:40.918689+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:42.074 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.918816+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e223: 8 total, 7 up, 8 in 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.922662+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.923267+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[77063]: cluster 2023-10-09T04:42:40.924102+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.550352+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v897: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.900034+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 153/1626 objects degraded (9.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.900568+0000 mon.a (mon.0) 1276 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.914299+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.075 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.915643+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: audit 2023-10-09T04:42:40.918689+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:42.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.918816+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e223: 8 total, 7 up, 8 in 2023-10-09T04:42:42.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.922662+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.923267+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:41 smithi082 bash[79733]: cluster 2023-10-09T04:42:40.924102+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.550352+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v897: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.900034+0000 mon.a (mon.0) 1275 : cluster 3 Health check update: Degraded data redundancy: 153/1626 objects degraded (9.410%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.900568+0000 mon.a (mon.0) 1276 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.914299+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.915643+0000 osd.4 (osd.4) 171 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: audit 2023-10-09T04:42:40.918689+0000 mon.a (mon.0) 1277 : audit 1 from='client.? 172.21.15.82:0/583273781' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.918816+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e223: 8 total, 7 up, 8 in 2023-10-09T04:42:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.922662+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.923267+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:41 smithi153 bash[59565]: cluster 2023-10-09T04:42:40.924102+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T04:42:42.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:42.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:42.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:42.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:42.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:42.466 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:42.491 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:42.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:42.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:42.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 16 7967 7951 165.631 118.125 0.00191609 0.0052849 2023-10-09T04:42:42.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:42.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:42.842 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:42.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:42.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:42.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:42.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:43.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:43.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:43.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:43.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:43.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:43.394 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:43.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:42:43.493 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:42:43.524 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:43.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:43.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 16 10533 10517 164.314 160.375 0.00342767 0.00607694 2023-10-09T04:42:43.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:43.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:43.919 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:43.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:43.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:44.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:44.063 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:43 smithi082 bash[77063]: cluster 2023-10-09T04:42:42.550931+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v899: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 64 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:44.064 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:43 smithi082 bash[79733]: cluster 2023-10-09T04:42:42.550931+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v899: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 64 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:44.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:44.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:43 smithi153 bash[59565]: cluster 2023-10-09T04:42:42.550931+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v899: 89 pgs: 5 active+undersized+degraded, 22 active+undersized, 62 active+clean; 37 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 64 op/s; 153/1626 objects degraded (9.410%) 2023-10-09T04:42:44.187 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:44.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:44.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:44.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:44.442 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:44.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:44.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:44.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:44.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:44.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 13361 13345 166.797 176.75 0.00520677 0.00598873 2023-10-09T04:42:44.735 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:44.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:44.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:44.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:44.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:44.935 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:44 smithi082 bash[77063]: audit 2023-10-09T04:42:44.282020+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:44.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.001 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:45.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:44 smithi153 bash[59565]: audit 2023-10-09T04:42:44.282020+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:45.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:45.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:44 smithi082 bash[79733]: audit 2023-10-09T04:42:44.282020+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:45.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:45.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:45.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:45.494 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:42:45.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:45.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:42:45.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:45.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:45.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:45.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 16 15839 15823 164.808 154.875 0.0121829 0.0060595 2023-10-09T04:42:45.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:45.891 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:45.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:45.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:45.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:45.937 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:45 smithi082 bash[77063]: cluster 2023-10-09T04:42:44.551558+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v900: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 451 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 2934/21531 objects degraded (13.627%) 2023-10-09T04:42:46.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:46.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:46.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:46.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:45 smithi153 bash[59565]: cluster 2023-10-09T04:42:44.551558+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v900: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 451 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 2934/21531 objects degraded (13.627%) 2023-10-09T04:42:46.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:46.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:45 smithi082 bash[79733]: cluster 2023-10-09T04:42:44.551558+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v900: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 451 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 2934/21531 objects degraded (13.627%) 2023-10-09T04:42:46.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:46.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:46.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:46.443 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:46.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:46.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:46.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:46.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:46.708 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:46.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 18135 18119 161.762 143.5 0.00980561 0.00617373 2023-10-09T04:42:46.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:46.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:46.944 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:46.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:46.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:47.017 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:47.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:46 smithi153 bash[59565]: cluster 2023-10-09T04:42:46.935535+0000 mon.a (mon.0) 1280 : cluster 3 Health check update: Degraded data redundancy: 4547/35904 objects degraded (12.664%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:47.110 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:47.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:46 smithi082 bash[77063]: cluster 2023-10-09T04:42:46.935535+0000 mon.a (mon.0) 1280 : cluster 3 Health check update: Degraded data redundancy: 4547/35904 objects degraded (12.664%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:47.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:46 smithi082 bash[79733]: cluster 2023-10-09T04:42:46.935535+0000 mon.a (mon.0) 1280 : cluster 3 Health check update: Degraded data redundancy: 4547/35904 objects degraded (12.664%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:47.241 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:47.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:47.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.369 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:47.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:47.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:47.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:47.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:47.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 20506 20490 160.063 148.188 0.00722568 0.00624132 2023-10-09T04:42:47.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:47.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:47.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:48.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:48.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:48.140 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:48.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:47 smithi153 bash[59565]: cluster 2023-10-09T04:42:46.552439+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v901: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 751 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4547/35904 objects degraded (12.664%) 2023-10-09T04:42:48.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:48.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:47 smithi082 bash[77063]: cluster 2023-10-09T04:42:46.552439+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v901: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 751 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4547/35904 objects degraded (12.664%) 2023-10-09T04:42:48.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:47 smithi082 bash[79733]: cluster 2023-10-09T04:42:46.552439+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v901: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 751 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4547/35904 objects degraded (12.664%) 2023-10-09T04:42:48.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:48.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:48.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:48.400 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:48.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:48.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:48.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:48.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:48.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:48.689 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:48.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 22764 22748 157.958 141.125 0.00630549 0.00632433 2023-10-09T04:42:48.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:48.824 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:48.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:48.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:48.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:49.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:49.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:49.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:49.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.347 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:49.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:49.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:49.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:49.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24634 8.d 8:b53b80c1:::benchmark_data_smithi082_175677_object24633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.689763+0000", 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0091150480000000006, 2023-10-09T04:42:49.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0091511479999999996, 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24634 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689763+0000", 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.703 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689763+0000", 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689761+0000", 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4946000000000001e-05 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689796+0000", 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3465999999999999e-05 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689810+0000", 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.126e-06 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.705 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689817+0000", 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7088e-05 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689864+0000", 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014538100000000001 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.690009+0000", 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000250282 2023-10-09T04:42:49.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 7", 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.690260+0000", 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024539290000000001 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692714+0000", 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029502 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.707 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24635 8.d 8:bc9b4ec3:::benchmark_data_smithi082_175677_object24634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.689833+0000", 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0090455919999999999, 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0091778899999999993, 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.708 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24635 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689833+0000", 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689833+0000", 2023-10-09T04:42:49.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689832+0000", 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9300999999999999e-05 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689861+0000", 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3599999999999998e-06 2023-10-09T04:42:49.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689869+0000", 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7650000000000001e-06 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.689874+0000", 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016266029999999999 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.691500+0000", 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000117407 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.691618+0000", 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014512800000000001 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 7", 2023-10-09T04:42:49.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.691763+0000", 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0021626029999999999 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.693925+0000", 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040924289999999999 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.713 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24640 8.8 8:19fd13e3:::benchmark_data_smithi082_175677_object24639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.692386+0000", 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0064926510000000003, 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0067722650000000004, 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24640 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.714 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692386+0000", 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692386+0000", 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.715 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692384+0000", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0346e-05 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692415+0000", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2347000000000001e-05 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692427+0000", 2023-10-09T04:42:49.716 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8909999999999998e-06 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692434+0000", 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8788e-05 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692493+0000", 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014641400000000001 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.717 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692639+0000", 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017276100000000001 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.692812+0000", 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0012379929999999999 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.694050+0000", 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016641539999999999 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24644 8.4 8:2770628a:::benchmark_data_smithi082_175677_object24643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.695249+0000", 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0036292579999999998, 2023-10-09T04:42:49.719 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040295180000000002, 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24644 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695249+0000", 2023-10-09T04:42:49.720 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695249+0000", 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695248+0000", 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1819000000000001e-05 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695280+0000", 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3614e-05 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695293+0000", 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8240000000000002e-06 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695300+0000", 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0630000000000001e-05 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695351+0000", 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000143879 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.723 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695495+0000", 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000168729 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 4", 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695663+0000", 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003539454 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.699203+0000", 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0039536090000000003 2023-10-09T04:42:49.724 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop_reply(client.35442.0:24640 8.8 e223/218)", 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.695331+0000", 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0035477249999999998, 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040282240000000004, 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "queued for pg", 2023-10-09T04:42:49.725 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695331+0000", 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695331+0000", 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.726 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695330+0000", 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4089999999999996e-06 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695335+0000", 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4060000000000005e-06 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.727 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695343+0000", 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695348+0000", 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7051999999999999e-05 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.728 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop(client.35442.0:24645 8.2 e223/218 8:44478d76:::benchmark_data_smithi082_175677_object24644:head v 223'1494, mlcod=223'1492)", 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.695793+0000", 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.003085562, 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0036215700000000002, 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695793+0000", 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.729 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695793+0000", 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695792+0000", 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.8592e-05 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695821+0000", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1565e-05 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695832+0000", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8249999999999999e-06 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695839+0000", 2023-10-09T04:42:49.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026440040000000001 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698483+0000", 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1548000000000001e-05 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698505+0000", 2023-10-09T04:42:49.732 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002711822 2023-10-09T04:42:49.733 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.733 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.733 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.733 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.733 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24646 8.4 8:28dccdfb:::benchmark_data_smithi082_175677_object24645:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.695775+0000", 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.003103055, 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0037141650000000002, 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24646 2023-10-09T04:42:49.734 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695775+0000", 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695775+0000", 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.735 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695774+0000", 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3812999999999994e-05 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695838+0000", 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3125000000000001e-05 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.736 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695851+0000", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3590000000000002e-06 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.695859+0000", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0012241190000000001 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697083+0000", 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013950400000000001 2023-10-09T04:42:49.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697222+0000", 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018210700000000001 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 4", 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697404+0000", 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020226829999999999 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.738 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.699427+0000", 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0036517089999999999 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_repop_reply(client.35442.0:24640 8.8 e223/218)", 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.696385+0000", 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.0024937990000000001, 2023-10-09T04:42:49.739 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.003197147, 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696385+0000", 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.740 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696385+0000", 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696384+0000", 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8269999999999997e-06 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696389+0000", 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0648e-05 2023-10-09T04:42:49.741 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696400+0000", 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.922e-06 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.696406+0000", 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031743769999999999 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.742 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.699581+0000", 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031961300000000002 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24648 8.0 8:0234ded5:::benchmark_data_smithi082_175677_object24647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.697161+0000", 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.00171729, 2023-10-09T04:42:49.743 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024999739999999999, 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24648 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.744 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697161+0000", 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697161+0000", 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.745 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697160+0000", 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.6625999999999999e-05 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697187+0000", 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6519999999999995e-06 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.746 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697196+0000", 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0900000000000004e-06 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.697201+0000", 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024201190000000001 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.699621+0000", 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024595110000000002 2023-10-09T04:42:49.747 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:24650 8.d 8:bcde6df9:::benchmark_data_smithi082_175677_object24649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:49.698560+0000", 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.00031801600000000001, 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001174551, 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:49.748 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 24650 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698560+0000", 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:42:49.749 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698560+0000", 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698559+0000", 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8310999999999999e-05 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698608+0000", 2023-10-09T04:42:49.750 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2556e-05 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698620+0000", 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8510000000000001e-06 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698627+0000", 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9988999999999999e-05 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.751 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698677+0000", 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012691199999999999 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698804+0000", 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016639000000000001 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 7", 2023-10-09T04:42:49.752 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:49.698970+0000", 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00040987899999999998 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: ], 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 10 2023-10-09T04:42:49.753 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:42:49.753 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:49.754 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 24675 24659 154.104 119.438 0.013485 0.00648277 2023-10-09T04:42:49.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:49.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:49.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:49.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:49.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:49.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:49] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T04:42:50.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:50.072 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:50.077 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:50.165 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:49 smithi153 bash[59565]: cluster 2023-10-09T04:42:48.553003+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v902: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 991 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 5310/47445 objects degraded (11.192%) 2023-10-09T04:42:50.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:50.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:50.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.462 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:49 smithi082 bash[77063]: cluster 2023-10-09T04:42:48.553003+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v902: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 991 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 5310/47445 objects degraded (11.192%) 2023-10-09T04:42:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:49 smithi082 bash[79733]: cluster 2023-10-09T04:42:48.553003+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v902: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 991 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 5310/47445 objects degraded (11.192%) 2023-10-09T04:42:50.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:50.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 15 26712 26697 151.673 127.375 0.00757357 0.00658731 2023-10-09T04:42:50.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:50.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:50.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:50.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:50.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:51.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:51.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:42:51.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:51.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:51.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:51.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:51.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:51.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:51.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:51.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:51.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 28589 28573 148.804 117.25 0.00954691 0.00671431 2023-10-09T04:42:51.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:51.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:51.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:51.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:51.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:52.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:52.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:52.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:51 smithi153 bash[59565]: cluster 2023-10-09T04:42:50.553832+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v903: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:52.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:52.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:52.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:52.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:52.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:52.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:51 smithi082 bash[77063]: cluster 2023-10-09T04:42:50.553832+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v903: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:52.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:51 smithi082 bash[79733]: cluster 2023-10-09T04:42:50.553832+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v903: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:52.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:52.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:52.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:52.711 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 30178 30162 144.996 99.3125 0.00668542 0.00688877 2023-10-09T04:42:52.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:52.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:52.853 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:52.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:52.982 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:52.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:52.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:53.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:53.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:53.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:53.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:53.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:53.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:53.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:53.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:53.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:53.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 31969 31953 142.634 111.938 0.00962565 0.00700592 2023-10-09T04:42:53.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:53.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:53.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:53.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:53.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:54.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:54.028 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:54 smithi153 bash[59565]: cluster 2023-10-09T04:42:52.554337+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v904: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:54.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:54.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:54.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:54.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:54.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:54.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:54.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:54.450 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:54 smithi082 bash[77063]: cluster 2023-10-09T04:42:52.554337+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v904: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:54 smithi082 bash[79733]: cluster 2023-10-09T04:42:52.554337+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v904: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 9112/71280 objects degraded (12.783%) 2023-10-09T04:42:54.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:54.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:54.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:54.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 16 33566 33550 139.779 99.8125 0.0048103 0.00714648 2023-10-09T04:42:54.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:54.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:54.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:54.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:54.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:54.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:54.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:55.083 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:55 smithi082 bash[77063]: cluster 2023-10-09T04:42:54.902652+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 9112/71280 objects degraded (12.783%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:55.083 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:55 smithi082 bash[79733]: cluster 2023-10-09T04:42:54.902652+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 9112/71280 objects degraded (12.783%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:55.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:55.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:55.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:55.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:55.383 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:55 smithi153 bash[59565]: cluster 2023-10-09T04:42:54.902652+0000 mon.a (mon.0) 1281 : cluster 3 Health check update: Degraded data redundancy: 9112/71280 objects degraded (12.783%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:55.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:55.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:55.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:55.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:55.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 35407 35391 138.233 115.062 0.0121978 0.00722841 2023-10-09T04:42:55.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:55.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:55.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:56.011 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:56 smithi082 bash[77063]: cluster 2023-10-09T04:42:54.555384+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v905: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s; 11476/88776 objects degraded (12.927%) 2023-10-09T04:42:56.012 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:56 smithi082 bash[79733]: cluster 2023-10-09T04:42:54.555384+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v905: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s; 11476/88776 objects degraded (12.927%) 2023-10-09T04:42:56.013 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:56.014 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:56.039 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:56.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:56.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:56.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:56.286 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:56 smithi153 bash[59565]: cluster 2023-10-09T04:42:54.555384+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v905: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s; 11476/88776 objects degraded (12.927%) 2023-10-09T04:42:56.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:56.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:56.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:56.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:56.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:56.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:56.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:56.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 16 37079 37063 136.248 104.5 0.00545825 0.0073338 2023-10-09T04:42:56.801 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:56.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:56.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:56.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:56.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:57.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:57.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:57.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:57.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:57.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:57.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:57.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:57.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:57.606 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:57.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:57.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 38581 38565 133.894 93.875 0.00212041 0.00745393 2023-10-09T04:42:57.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:57.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:57.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:57.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:57.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:58.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:58.038 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:58.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:58.169 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:58.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:58.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:58 smithi082 bash[77063]: cluster 2023-10-09T04:42:56.556074+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v906: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 12562/98619 objects degraded (12.738%) 2023-10-09T04:42:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:58 smithi082 bash[79733]: cluster 2023-10-09T04:42:56.556074+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v906: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 12562/98619 objects degraded (12.738%) 2023-10-09T04:42:58.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:58.285 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:58 smithi153 bash[59565]: cluster 2023-10-09T04:42:56.556074+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v906: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s; 12562/98619 objects degraded (12.738%) 2023-10-09T04:42:58.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:58.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:58.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:58.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:58.566 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:58.669 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:58.701 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:58.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 16 39905 39889 131.201 82.75 0.00425505 0.00761314 2023-10-09T04:42:58.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:58.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:58.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:58.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:58.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:59.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:42:59 smithi082 bash[77063]: cluster 2023-10-09T04:42:58.556688+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v907: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 13084/106206 objects degraded (12.319%) 2023-10-09T04:42:59.026 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:42:59 smithi082 bash[79733]: cluster 2023-10-09T04:42:58.556688+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v907: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 13084/106206 objects degraded (12.319%) 2023-10-09T04:42:59.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.089 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:59.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.214 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:42:59 smithi153 bash[59565]: cluster 2023-10-09T04:42:58.556688+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v907: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 13084/106206 objects degraded (12.319%) 2023-10-09T04:42:59.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:59.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:59.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:59.466 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.492 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:59.597 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:59.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:42:59.711936+0000 min lat: 0.00125881 max lat: 0.429926 avg lat: 0.00768785 2023-10-09T04:42:59.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:42:59.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 16 41614 41598 129.981 106.812 0.0128842 0.00768785 2023-10-09T04:42:59.725 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:42:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:42:59] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T04:42:59.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:59.853 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:59.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:42:59.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:59.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:00.027 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:00 smithi082 bash[77063]: audit 2023-10-09T04:42:59.282131+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:00.027 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:00 smithi082 bash[77063]: cluster 2023-10-09T04:42:59.903965+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 13084/106206 objects degraded (12.319%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:00.027 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:00 smithi082 bash[79733]: audit 2023-10-09T04:42:59.282131+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:00.028 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:00 smithi082 bash[79733]: cluster 2023-10-09T04:42:59.903965+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 13084/106206 objects degraded (12.319%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:00.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:00.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:00.343 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:00 smithi153 bash[59565]: audit 2023-10-09T04:42:59.282131+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:00.343 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:00 smithi153 bash[59565]: cluster 2023-10-09T04:42:59.903965+0000 mon.a (mon.0) 1283 : cluster 3 Health check update: Degraded data redundancy: 13084/106206 objects degraded (12.319%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:00.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:00.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:00.514 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:00.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:00.646 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:00.678 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:00.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 43346 43330 128.946 108.25 0.00500381 0.00774358 2023-10-09T04:43:00.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:00.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:00.912 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:00.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:01.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:01.094 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:01 smithi082 bash[77063]: cluster 2023-10-09T04:43:00.557318+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v908: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:01.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:01 smithi153 bash[59565]: cluster 2023-10-09T04:43:00.557318+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v908: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:01.195 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:01.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:01.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:01.328 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:01.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:01.516 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:01 smithi082 bash[79733]: cluster 2023-10-09T04:43:00.557318+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v908: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:01.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:01.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:01.690 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:01.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 44781 44765 127.161 89.6875 0.0156549 0.00785901 2023-10-09T04:43:01.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:01.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:01.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:01.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:01.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:02.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:02.223 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:02.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:02.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:02.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:02.506 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:02.547 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:43:02.547 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:43:02.548 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:43:02.548 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:43:02.548 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:43:02.610 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:02.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:02.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 16 46346 46330 125.885 97.8125 0.011827 0.00793889 2023-10-09T04:43:02.753 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:02.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:02.914 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:02.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:03.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:03.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:03.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:03.121 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:43:03.130 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:43:03.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:03.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:03.234 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:43:03.274 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:43:03.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:03.317 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:43:03.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:03.366 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:43:03.410 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:43:03.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:03.449 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:43:03.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:03.482 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.483 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.484 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:43:03.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:03.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:03.640 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[77063]: cluster 2023-10-09T04:43:02.557997+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v909: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[77063]: audit 2023-10-09T04:43:03.168143+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.82:0/1709176824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[77063]: audit 2023-10-09T04:43:03.567814+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.82:0/3817157925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[77063]: audit 2023-10-09T04:43:03.568464+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[79733]: cluster 2023-10-09T04:43:02.557997+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v909: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:03.670 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[79733]: audit 2023-10-09T04:43:03.168143+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.82:0/1709176824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[79733]: audit 2023-10-09T04:43:03.567814+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.82:0/3817157925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.671 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:03 smithi082 bash[79733]: audit 2023-10-09T04:43:03.568464+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:03.712 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 47689 47673 124.137 83.9375 0.0225629 0.00804924 2023-10-09T04:43:03.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:03.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:43:03.825 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:03.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:03.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:03 smithi153 bash[59565]: cluster 2023-10-09T04:43:02.557997+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v909: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 15611/122592 objects degraded (12.734%) 2023-10-09T04:43:03.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:03 smithi153 bash[59565]: audit 2023-10-09T04:43:03.168143+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.82:0/1709176824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:03 smithi153 bash[59565]: audit 2023-10-09T04:43:03.567814+0000 mon.c (mon.2) 94 : audit 1 from='client.? 172.21.15.82:0/3817157925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.874 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:03 smithi153 bash[59565]: audit 2023-10-09T04:43:03.568464+0000 mon.a (mon.0) 1284 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:03.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:03.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:03.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:04.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:04.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:04.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:04.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:04.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:04.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:04.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:04.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:04.514 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:04.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:04.619 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: audit 2023-10-09T04:43:03.106059+0000 mgr.y (mgr.34107) 868 : audit 0 from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:04.619 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.614387+0000 mon.a (mon.0) 1285 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:04.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:04.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:04.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 49208 49192 122.968 94.9375 0.00635307 0.0081271 2023-10-09T04:43:04.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:04.811 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:04.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:04.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: audit 2023-10-09T04:43:03.106059+0000 mgr.y (mgr.34107) 868 : audit 0 from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.614387+0000 mon.a (mon.0) 1285 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.625372+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.625794+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: audit 2023-10-09T04:43:03.628176+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.628302+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e224: 8 total, 7 up, 8 in 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.632203+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:04 smithi153 bash[59565]: cluster 2023-10-09T04:43:03.635317+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:04.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.625372+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.625794+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: audit 2023-10-09T04:43:03.628176+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:04.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.628302+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e224: 8 total, 7 up, 8 in 2023-10-09T04:43:04.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.632203+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[77063]: cluster 2023-10-09T04:43:03.635317+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: audit 2023-10-09T04:43:03.106059+0000 mgr.y (mgr.34107) 868 : audit 0 from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.614387+0000 mon.a (mon.0) 1285 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:04.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.625372+0000 osd.4 (osd.4) 172 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.625794+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: audit 2023-10-09T04:43:03.628176+0000 mon.a (mon.0) 1286 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:04.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.628302+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e224: 8 total, 7 up, 8 in 2023-10-09T04:43:04.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.632203+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:04.984 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:04 smithi082 bash[79733]: cluster 2023-10-09T04:43:03.635317+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T04:43:05.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:05.066 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:05.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:05.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:05.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:05.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:05.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:05.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:05.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:05.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 50759 50743 121.967 96.9375 0.0115414 0.00819401 2023-10-09T04:43:05.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:05.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:05.866 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:05.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:05 smithi153 bash[59565]: cluster 2023-10-09T04:43:04.558681+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v911: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 17546/137283 objects degraded (12.781%) 2023-10-09T04:43:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:05 smithi153 bash[59565]: cluster 2023-10-09T04:43:04.905469+0000 mon.a (mon.0) 1288 : cluster 3 Health check update: Degraded data redundancy: 17546/137283 objects degraded (12.781%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:05.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:05 smithi082 bash[77063]: cluster 2023-10-09T04:43:04.558681+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v911: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 17546/137283 objects degraded (12.781%) 2023-10-09T04:43:05.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:05 smithi082 bash[77063]: cluster 2023-10-09T04:43:04.905469+0000 mon.a (mon.0) 1288 : cluster 3 Health check update: Degraded data redundancy: 17546/137283 objects degraded (12.781%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:05 smithi082 bash[79733]: cluster 2023-10-09T04:43:04.558681+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v911: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 17546/137283 objects degraded (12.781%) 2023-10-09T04:43:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:05 smithi082 bash[79733]: cluster 2023-10-09T04:43:04.905469+0000 mon.a (mon.0) 1288 : cluster 3 Health check update: Degraded data redundancy: 17546/137283 objects degraded (12.781%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:06.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:06.061 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":909,"stamp":"2023-10-09T04:43:02.557510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2681018291,"num_objects":40864,"num_object_clones":0,"num_object_copies":122592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40864,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":41942,"num_write_kb":2628663,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":8398292,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42006,"ondisk_log_size":42006,"up":240,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":265,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10167500,"kb_used_data":7088460,"kb_used_omap":46,"kb_used_meta":3078929,"kb_avail":739793716,"statfs":{"total":767960285184,"available":757548765184,"internally_reserved":0,"allocated":7258583040,"data_stored":7252730753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47544,"internal_metadata":3152823880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,6,6,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1120927744,"num_objects":17104,"num_object_clones":0,"num_object_copies":51312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6499,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17104,"num_write_kb":1094656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004360"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":210,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:36.385108+0000","last_change":"2023-10-09T04:42:20.925157+0000","last_active":"2023-10-09T04:42:36.385108+0000","last_peered":"2023-10-09T04:42:36.385108+0000","last_clean":"2023-10-09T04:42:05.106706+0000","last_became_active":"2023-10-09T04:42:20.925157+0000","last_became_peered":"2023-10-09T04:42:20.925157+0000","last_unstale":"2023-10-09T04:42:36.385108+0000","last_undegraded":"2023-10-09T04:42:36.385108+0000","last_fullsized":"2023-10-09T04:42:20.923392+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:36.386388+0000","last_change":"2023-10-09T04:42:20.925882+0000","last_active":"2023-10-09T04:42:36.386388+0000","last_peered":"2023-10-09T04:42:36.386388+0000","last_clean":"2023-10-09T04:22:50.689368+0000","last_became_active":"2023-10-09T04:42:20.925882+0000","last_became_peered":"2023-10-09T04:42:20.925882+0000","last_unstale":"2023-10-09T04:42:36.386388+0000","last_undegraded":"2023-10-09T04:42:36.386388+0000","last_fullsized":"2023-10-09T04:42:20.924199+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":216,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.235096+0000","last_change":"2023-10-09T04:38:49.687475+0000","last_active":"2023-10-09T04:42:37.235096+0000","last_peered":"2023-10-09T04:42:37.235096+0000","last_clean":"2023-10-09T04:42:37.235096+0000","last_became_active":"2023-10-09T04:38:49.687077+0000","last_became_peered":"2023-10-09T04:38:49.687077+0000","last_unstale":"2023-10-09T04:42:37.235096+0000","last_undegraded":"2023-10-09T04:42:37.235096+0000","last_fullsized":"2023-10-09T04:42:37.235096+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:41.450502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":198,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.186010+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:42:37.186010+0000","last_peered":"2023-10-09T04:42:37.186010+0000","last_clean":"2023-10-09T04:42:37.186010+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:42:37.186010+0000","last_undegraded":"2023-10-09T04:42:37.186010+0000","last_fullsized":"2023-10-09T04:42:37.186010+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.184893+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:42:37.184893+0000","last_peered":"2023-10-09T04:42:37.184893+0000","last_clean":"2023-10-09T04:42:37.184893+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:42:37.184893+0000","last_undegraded":"2023-10-09T04:42:37.184893+0000","last_fullsized":"2023-10-09T04:42:37.184893+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.163063+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:42:37.163063+0000","last_peered":"2023-10-09T04:42:37.163063+0000","last_clean":"2023-10-09T04:42:37.163063+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:42:37.163063+0000","last_undegraded":"2023-10-09T04:42:37.163063+0000","last_fullsized":"2023-10-09T04:42:37.163063+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.184680+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:42:37.184680+0000","last_peered":"2023-10-09T04:42:37.184680+0000","last_clean":"2023-10-09T04:42:37.184680+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:42:37.184680+0000","last_undegraded":"2023-10-09T04:42:37.184680+0000","last_fullsized":"2023-10-09T04:42:37.184680+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.380480+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:42:36.380480+0000","last_peered":"2023-10-09T04:42:36.380480+0000","last_clean":"2023-10-09T04:42:36.380480+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:42:36.380480+0000","last_undegraded":"2023-10-09T04:42:36.380480+0000","last_fullsized":"2023-10-09T04:42:36.380480+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":164,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385732+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:42:36.385732+0000","last_peered":"2023-10-09T04:42:36.385732+0000","last_clean":"2023-10-09T04:42:36.385732+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:42:36.385732+0000","last_undegraded":"2023-10-09T04:42:36.385732+0000","last_fullsized":"2023-10-09T04:42:36.385732+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260325+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:42:37.260325+0000","last_peered":"2023-10-09T04:42:37.260325+0000","last_clean":"2023-10-09T04:42:37.260325+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:42:37.260325+0000","last_undegraded":"2023-10-09T04:42:37.260325+0000","last_fullsized":"2023-10-09T04:42:37.260325+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":44,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260700+0000","last_change":"2023-10-09T04:40:47.996370+0000","last_active":"2023-10-09T04:42:37.260700+0000","last_peered":"2023-10-09T04:42:37.260700+0000","last_clean":"2023-10-09T04:42:37.260700+0000","last_became_active":"2023-10-09T04:40:47.817084+0000","last_became_peered":"2023-10-09T04:40:47.817084+0000","last_unstale":"2023-10-09T04:42:37.260700+0000","last_undegraded":"2023-10-09T04:42:37.260700+0000","last_fullsized":"2023-10-09T04:42:37.260700+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":195,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.235189+0000","last_change":"2023-10-09T04:42:20.922762+0000","last_active":"2023-10-09T04:42:37.235189+0000","last_peered":"2023-10-09T04:42:37.235189+0000","last_clean":"2023-10-09T04:42:04.812847+0000","last_became_active":"2023-10-09T04:42:20.922762+0000","last_became_peered":"2023-10-09T04:42:20.922762+0000","last_unstale":"2023-10-09T04:42:37.235189+0000","last_undegraded":"2023-10-09T04:42:37.235189+0000","last_fullsized":"2023-10-09T04:42:20.914425+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:46.474048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":248,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386812+0000","last_change":"2023-10-09T04:40:47.863039+0000","last_active":"2023-10-09T04:42:36.386812+0000","last_peered":"2023-10-09T04:42:36.386812+0000","last_clean":"2023-10-09T04:42:36.386812+0000","last_became_active":"2023-10-09T04:40:47.819279+0000","last_became_peered":"2023-10-09T04:40:47.819279+0000","last_unstale":"2023-10-09T04:42:36.386812+0000","last_undegraded":"2023-10-09T04:42:36.386812+0000","last_fullsized":"2023-10-09T04:42:36.386812+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386374+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:42:36.386374+0000","last_peered":"2023-10-09T04:42:36.386374+0000","last_clean":"2023-10-09T04:42:36.386374+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:42:36.386374+0000","last_undegraded":"2023-10-09T04:42:36.386374+0000","last_fullsized":"2023-10-09T04:42:36.386374+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":50,"reported_epoch":222,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:37.259501+0000","last_change":"2023-10-09T04:42:20.928746+0000","last_active":"2023-10-09T04:42:37.259501+0000","last_peered":"2023-10-09T04:42:37.259501+0000","last_clean":"2023-10-09T04:42:04.815993+0000","last_became_active":"2023-10-09T04:42:20.928746+0000","last_became_peered":"2023-10-09T04:42:20.928746+0000","last_unstale":"2023-10-09T04:42:37.259501+0000","last_undegraded":"2023-10-09T04:42:20.924100+0000","last_fullsized":"2023-10-09T04:42:20.923957+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:40.925493+0000","last_change":"2023-10-09T04:40:47.816775+0000","last_active":"2023-10-09T04:42:40.925493+0000","last_peered":"2023-10-09T04:42:40.925493+0000","last_clean":"2023-10-09T04:42:40.925493+0000","last_became_active":"2023-10-09T04:40:47.816424+0000","last_became_peered":"2023-10-09T04:40:47.816424+0000","last_unstale":"2023-10-09T04:42:40.925493+0000","last_undegraded":"2023-10-09T04:42:40.925493+0000","last_fullsized":"2023-10-09T04:42:40.925493+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:52.923977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":196,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185372+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:42:37.185372+0000","last_peered":"2023-10-09T04:42:37.185372+0000","last_clean":"2023-10-09T04:42:37.185372+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:42:37.185372+0000","last_undegraded":"2023-10-09T04:42:37.185372+0000","last_fullsized":"2023-10-09T04:42:37.185372+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":173,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386170+0000","last_change":"2023-10-09T04:40:47.820893+0000","last_active":"2023-10-09T04:42:36.386170+0000","last_peered":"2023-10-09T04:42:36.386170+0000","last_clean":"2023-10-09T04:42:36.386170+0000","last_became_active":"2023-10-09T04:40:47.820570+0000","last_became_peered":"2023-10-09T04:40:47.820570+0000","last_unstale":"2023-10-09T04:42:36.386170+0000","last_undegraded":"2023-10-09T04:42:36.386170+0000","last_fullsized":"2023-10-09T04:42:36.386170+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":38,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.259906+0000","last_change":"2023-10-09T04:40:47.823962+0000","last_active":"2023-10-09T04:42:37.259906+0000","last_peered":"2023-10-09T04:42:37.259906+0000","last_clean":"2023-10-09T04:42:37.259906+0000","last_became_active":"2023-10-09T04:40:47.823509+0000","last_became_peered":"2023-10-09T04:40:47.823509+0000","last_unstale":"2023-10-09T04:42:37.259906+0000","last_undegraded":"2023-10-09T04:42:37.259906+0000","last_fullsized":"2023-10-09T04:42:37.259906+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":196,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.186087+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:42:37.186087+0000","last_peered":"2023-10-09T04:42:37.186087+0000","last_clean":"2023-10-09T04:42:37.186087+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:42:37.186087+0000","last_undegraded":"2023-10-09T04:42:37.186087+0000","last_fullsized":"2023-10-09T04:42:37.186087+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":187,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.162858+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:42:37.162858+0000","last_peered":"2023-10-09T04:42:37.162858+0000","last_clean":"2023-10-09T04:42:37.162858+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:42:37.162858+0000","last_undegraded":"2023-10-09T04:42:37.162858+0000","last_fullsized":"2023-10-09T04:42:37.162858+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.381527+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:42:36.381527+0000","last_peered":"2023-10-09T04:42:36.381527+0000","last_clean":"2023-10-09T04:42:36.381527+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:42:36.381527+0000","last_undegraded":"2023-10-09T04:42:36.381527+0000","last_fullsized":"2023-10-09T04:42:36.381527+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"223'2612","reported_seq":2622,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:43:00.099400+0000","last_change":"2023-10-09T04:42:39.716801+0000","last_active":"2023-10-09T04:43:00.099400+0000","last_peered":"2023-10-09T04:43:00.099400+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.933694+0000","last_became_peered":"2023-10-09T04:42:20.933694+0000","last_unstale":"2023-10-09T04:43:00.099400+0000","last_undegraded":"2023-10-09T04:42:39.716575+0000","last_fullsized":"2023-10-09T04:42:20.930067+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2612,"log_dups_size":0,"ondisk_log_size":2612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171180032,"num_objects":2612,"num_object_clones":0,"num_object_copies":7836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2612,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2612,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2612}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":42,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260713+0000","last_change":"2023-10-09T04:40:48.230655+0000","last_active":"2023-10-09T04:42:37.260713+0000","last_peered":"2023-10-09T04:42:37.260713+0000","last_clean":"2023-10-09T04:42:37.260713+0000","last_became_active":"2023-10-09T04:40:47.830826+0000","last_became_peered":"2023-10-09T04:40:47.830826+0000","last_unstale":"2023-10-09T04:42:37.260713+0000","last_undegraded":"2023-10-09T04:42:37.260713+0000","last_fullsized":"2023-10-09T04:42:37.260713+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386573+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:42:36.386573+0000","last_peered":"2023-10-09T04:42:36.386573+0000","last_clean":"2023-10-09T04:42:36.386573+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:42:36.386573+0000","last_undegraded":"2023-10-09T04:42:36.386573+0000","last_fullsized":"2023-10-09T04:42:36.386573+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"223'2581","reported_seq":2594,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:58.205054+0000","last_change":"2023-10-09T04:42:20.927819+0000","last_active":"2023-10-09T04:42:58.205054+0000","last_peered":"2023-10-09T04:42:58.205054+0000","last_clean":"2023-10-09T04:42:58.205054+0000","last_became_active":"2023-10-09T04:42:20.927473+0000","last_became_peered":"2023-10-09T04:42:20.927473+0000","last_unstale":"2023-10-09T04:42:58.205054+0000","last_undegraded":"2023-10-09T04:42:58.205054+0000","last_fullsized":"2023-10-09T04:42:58.205054+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2581,"log_dups_size":0,"ondisk_log_size":2581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2581,"num_write_kb":165184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":42,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260737+0000","last_change":"2023-10-09T04:40:48.463697+0000","last_active":"2023-10-09T04:42:37.260737+0000","last_peered":"2023-10-09T04:42:37.260737+0000","last_clean":"2023-10-09T04:42:37.260737+0000","last_became_active":"2023-10-09T04:40:47.913470+0000","last_became_peered":"2023-10-09T04:40:47.913470+0000","last_unstale":"2023-10-09T04:42:37.260737+0000","last_undegraded":"2023-10-09T04:42:37.260737+0000","last_fullsized":"2023-10-09T04:42:37.260737+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385251+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:42:36.385251+0000","last_peered":"2023-10-09T04:42:36.385251+0000","last_clean":"2023-10-09T04:42:36.385251+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:42:36.385251+0000","last_undegraded":"2023-10-09T04:42:36.385251+0000","last_fullsized":"2023-10-09T04:42:36.385251+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"223'2440","reported_seq":2454,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:58.422131+0000","last_change":"2023-10-09T04:42:20.927851+0000","last_active":"2023-10-09T04:42:58.422131+0000","last_peered":"2023-10-09T04:42:58.422131+0000","last_clean":"2023-10-09T04:42:58.422131+0000","last_became_active":"2023-10-09T04:42:20.927524+0000","last_became_peered":"2023-10-09T04:42:20.927524+0000","last_unstale":"2023-10-09T04:42:58.422131+0000","last_undegraded":"2023-10-09T04:42:58.422131+0000","last_fullsized":"2023-10-09T04:42:58.422131+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2440,"log_dups_size":0,"ondisk_log_size":2440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:54.995085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2440,"num_write_kb":156160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":42,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260592+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:42:37.260592+0000","last_peered":"2023-10-09T04:42:37.260592+0000","last_clean":"2023-10-09T04:42:37.260592+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:42:37.260592+0000","last_undegraded":"2023-10-09T04:42:37.260592+0000","last_fullsized":"2023-10-09T04:42:37.260592+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":207,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.164110+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:42:37.164110+0000","last_peered":"2023-10-09T04:42:37.164110+0000","last_clean":"2023-10-09T04:42:37.164110+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:42:37.164110+0000","last_undegraded":"2023-10-09T04:42:37.164110+0000","last_fullsized":"2023-10-09T04:42:37.164110+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"223'2545","reported_seq":2559,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:59.073261+0000","last_change":"2023-10-09T04:42:20.922659+0000","last_active":"2023-10-09T04:42:59.073261+0000","last_peered":"2023-10-09T04:42:59.073261+0000","last_clean":"2023-10-09T04:42:59.073261+0000","last_became_active":"2023-10-09T04:42:20.922288+0000","last_became_peered":"2023-10-09T04:42:20.922288+0000","last_unstale":"2023-10-09T04:42:59.073261+0000","last_undegraded":"2023-10-09T04:42:59.073261+0000","last_fullsized":"2023-10-09T04:42:59.073261+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2545,"log_dups_size":0,"ondisk_log_size":2545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:59.163318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166789120,"num_objects":2545,"num_object_clones":0,"num_object_copies":7635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":162880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":156,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385616+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:42:36.385616+0000","last_peered":"2023-10-09T04:42:36.385616+0000","last_clean":"2023-10-09T04:42:36.385616+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:42:36.385616+0000","last_undegraded":"2023-10-09T04:42:36.385616+0000","last_fullsized":"2023-10-09T04:42:36.385616+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":198,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.184603+0000","last_change":"2023-10-09T04:42:20.926820+0000","last_active":"2023-10-09T04:42:37.184603+0000","last_peered":"2023-10-09T04:42:37.184603+0000","last_clean":"2023-10-09T04:42:04.818258+0000","last_became_active":"2023-10-09T04:42:20.926820+0000","last_became_peered":"2023-10-09T04:42:20.926820+0000","last_unstale":"2023-10-09T04:42:37.184603+0000","last_undegraded":"2023-10-09T04:42:37.184603+0000","last_fullsized":"2023-10-09T04:42:20.918277+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:35.573480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"223'2535","reported_seq":2548,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:43:00.091959+0000","last_change":"2023-10-09T04:42:20.935850+0000","last_active":"2023-10-09T04:43:00.091959+0000","last_peered":"2023-10-09T04:43:00.091959+0000","last_clean":"2023-10-09T04:43:00.091959+0000","last_became_active":"2023-10-09T04:42:20.935238+0000","last_became_peered":"2023-10-09T04:42:20.935238+0000","last_unstale":"2023-10-09T04:43:00.091959+0000","last_undegraded":"2023-10-09T04:43:00.091959+0000","last_fullsized":"2023-10-09T04:43:00.091959+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2535,"log_dups_size":0,"ondisk_log_size":2535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2535,"num_write_kb":162240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":142,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185355+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:42:37.185355+0000","last_peered":"2023-10-09T04:42:37.185355+0000","last_clean":"2023-10-09T04:42:37.185355+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:42:37.185355+0000","last_undegraded":"2023-10-09T04:42:37.185355+0000","last_fullsized":"2023-10-09T04:42:37.185355+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386873+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:42:36.386873+0000","last_peered":"2023-10-09T04:42:36.386873+0000","last_clean":"2023-10-09T04:42:36.386873+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:42:36.386873+0000","last_undegraded":"2023-10-09T04:42:36.386873+0000","last_fullsized":"2023-10-09T04:42:36.386873+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"223'2591","reported_seq":2602,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:59.062607+0000","last_change":"2023-10-09T04:42:39.730855+0000","last_active":"2023-10-09T04:42:59.062607+0000","last_peered":"2023-10-09T04:42:59.062607+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.922693+0000","last_became_peered":"2023-10-09T04:42:20.922693+0000","last_unstale":"2023-10-09T04:42:59.062607+0000","last_undegraded":"2023-10-09T04:42:39.713503+0000","last_fullsized":"2023-10-09T04:42:20.911478+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2591,"log_dups_size":0,"ondisk_log_size":2591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:06.219357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2591,"num_write_kb":165824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":2591}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":190,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.164000+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:42:37.164000+0000","last_peered":"2023-10-09T04:42:37.164000+0000","last_clean":"2023-10-09T04:42:37.164000+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:42:37.164000+0000","last_undegraded":"2023-10-09T04:42:37.164000+0000","last_fullsized":"2023-10-09T04:42:37.164000+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.235850+0000","last_change":"2023-10-09T04:42:20.921913+0000","last_active":"2023-10-09T04:42:37.235850+0000","last_peered":"2023-10-09T04:42:37.235850+0000","last_clean":"2023-10-09T04:33:01.733237+0000","last_became_active":"2023-10-09T04:42:20.921913+0000","last_became_peered":"2023-10-09T04:42:20.921913+0000","last_unstale":"2023-10-09T04:42:37.235850+0000","last_undegraded":"2023-10-09T04:42:37.235850+0000","last_fullsized":"2023-10-09T04:42:20.912151+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:48.367063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"223'2625","reported_seq":2638,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:43:00.300546+0000","last_change":"2023-10-09T04:42:20.932304+0000","last_active":"2023-10-09T04:43:00.300546+0000","last_peered":"2023-10-09T04:43:00.300546+0000","last_clean":"2023-10-09T04:43:00.300546+0000","last_became_active":"2023-10-09T04:42:20.931925+0000","last_became_peered":"2023-10-09T04:42:20.931925+0000","last_unstale":"2023-10-09T04:43:00.300546+0000","last_undegraded":"2023-10-09T04:43:00.300546+0000","last_fullsized":"2023-10-09T04:43:00.300546+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2625,"log_dups_size":0,"ondisk_log_size":2625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172032000,"num_objects":2625,"num_object_clones":0,"num_object_copies":7875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2625,"num_write_kb":168000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.259947+0000","last_change":"2023-10-09T04:42:20.925082+0000","last_active":"2023-10-09T04:42:37.259947+0000","last_peered":"2023-10-09T04:42:37.259947+0000","last_clean":"2023-10-09T04:40:15.045258+0000","last_became_active":"2023-10-09T04:42:20.925082+0000","last_became_peered":"2023-10-09T04:42:20.925082+0000","last_unstale":"2023-10-09T04:42:37.259947+0000","last_undegraded":"2023-10-09T04:42:37.259947+0000","last_fullsized":"2023-10-09T04:42:20.922246+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386625+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:42:36.386625+0000","last_peered":"2023-10-09T04:42:36.386625+0000","last_clean":"2023-10-09T04:42:36.386625+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:42:36.386625+0000","last_undegraded":"2023-10-09T04:42:36.386625+0000","last_fullsized":"2023-10-09T04:42:36.386625+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"223'2544","reported_seq":2558,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:59.051189+0000","last_change":"2023-10-09T04:42:20.933001+0000","last_active":"2023-10-09T04:42:59.051189+0000","last_peered":"2023-10-09T04:42:59.051189+0000","last_clean":"2023-10-09T04:42:59.051189+0000","last_became_active":"2023-10-09T04:42:20.932460+0000","last_became_peered":"2023-10-09T04:42:20.932460+0000","last_unstale":"2023-10-09T04:42:59.051189+0000","last_undegraded":"2023-10-09T04:42:59.051189+0000","last_fullsized":"2023-10-09T04:42:59.051189+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2544,"log_dups_size":0,"ondisk_log_size":2544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:13.549989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166723584,"num_objects":2544,"num_object_clones":0,"num_object_copies":7632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2544,"num_write_kb":162816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":196,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.380260+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:42:36.380260+0000","last_peered":"2023-10-09T04:42:36.380260+0000","last_clean":"2023-10-09T04:42:36.380260+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:42:36.380260+0000","last_undegraded":"2023-10-09T04:42:36.380260+0000","last_fullsized":"2023-10-09T04:42:36.380260+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.235924+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:42:37.235924+0000","last_peered":"2023-10-09T04:42:37.235924+0000","last_clean":"2023-10-09T04:42:37.235924+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:42:37.235924+0000","last_undegraded":"2023-10-09T04:42:37.235924+0000","last_fullsized":"2023-10-09T04:42:37.235924+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"223'2492","reported_seq":2503,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:58.433450+0000","last_change":"2023-10-09T04:42:39.717793+0000","last_active":"2023-10-09T04:42:58.433450+0000","last_peered":"2023-10-09T04:42:58.433450+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.928238+0000","last_became_peered":"2023-10-09T04:42:20.928238+0000","last_unstale":"2023-10-09T04:42:58.433450+0000","last_undegraded":"2023-10-09T04:42:39.716102+0000","last_fullsized":"2023-10-09T04:42:20.920208+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2492,"log_dups_size":0,"ondisk_log_size":2492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:20.684642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2492,"num_write_kb":159488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2492}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":674,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.235683+0000","last_change":"2023-10-09T04:41:04.902111+0000","last_active":"2023-10-09T04:42:37.235683+0000","last_peered":"2023-10-09T04:42:37.235683+0000","last_clean":"2023-10-09T04:42:37.235683+0000","last_became_active":"2023-10-09T04:41:04.901604+0000","last_became_peered":"2023-10-09T04:41:04.901604+0000","last_unstale":"2023-10-09T04:42:37.235683+0000","last_undegraded":"2023-10-09T04:42:37.235683+0000","last_fullsized":"2023-10-09T04:42:37.235683+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:55.374183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:36.386932+0000","last_change":"2023-10-09T04:42:20.934857+0000","last_active":"2023-10-09T04:42:36.386932+0000","last_peered":"2023-10-09T04:42:36.386932+0000","last_clean":"2023-10-09T04:42:04.816814+0000","last_became_active":"2023-10-09T04:42:20.934857+0000","last_became_peered":"2023-10-09T04:42:20.934857+0000","last_unstale":"2023-10-09T04:42:36.386932+0000","last_undegraded":"2023-10-09T04:42:36.386932+0000","last_fullsized":"2023-10-09T04:42:20.927835+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.235216+0000","last_change":"2023-10-09T04:38:49.685222+0000","last_active":"2023-10-09T04:42:37.235216+0000","last_peered":"2023-10-09T04:42:37.235216+0000","last_clean":"2023-10-09T04:42:37.235216+0000","last_became_active":"2023-10-09T04:38:49.684705+0000","last_became_peered":"2023-10-09T04:38:49.684705+0000","last_unstale":"2023-10-09T04:42:37.235216+0000","last_undegraded":"2023-10-09T04:42:37.235216+0000","last_fullsized":"2023-10-09T04:42:37.235216+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:42.039283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"223'2602","reported_seq":2614,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:43:00.100752+0000","last_change":"2023-10-09T04:42:20.935270+0000","last_active":"2023-10-09T04:43:00.100752+0000","last_peered":"2023-10-09T04:43:00.100752+0000","last_clean":"2023-10-09T04:43:00.100752+0000","last_became_active":"2023-10-09T04:42:20.934977+0000","last_became_peered":"2023-10-09T04:42:20.934977+0000","last_unstale":"2023-10-09T04:43:00.100752+0000","last_undegraded":"2023-10-09T04:43:00.100752+0000","last_fullsized":"2023-10-09T04:43:00.100752+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2602,"log_dups_size":0,"ondisk_log_size":2602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170524672,"num_objects":2602,"num_object_clones":0,"num_object_copies":7806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2602,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":685,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386066+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:42:36.386066+0000","last_peered":"2023-10-09T04:42:36.386066+0000","last_clean":"2023-10-09T04:42:36.386066+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:42:36.386066+0000","last_undegraded":"2023-10-09T04:42:36.386066+0000","last_fullsized":"2023-10-09T04:42:36.386066+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":203,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386255+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:42:36.386255+0000","last_peered":"2023-10-09T04:42:36.386255+0000","last_clean":"2023-10-09T04:42:36.386255+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:42:36.386255+0000","last_undegraded":"2023-10-09T04:42:36.386255+0000","last_fullsized":"2023-10-09T04:42:36.386255+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":196,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.234993+0000","last_change":"2023-10-09T04:42:20.917299+0000","last_active":"2023-10-09T04:42:37.234993+0000","last_peered":"2023-10-09T04:42:37.234993+0000","last_clean":"2023-10-09T04:42:04.812997+0000","last_became_active":"2023-10-09T04:42:20.917299+0000","last_became_peered":"2023-10-09T04:42:20.917299+0000","last_unstale":"2023-10-09T04:42:37.234993+0000","last_undegraded":"2023-10-09T04:42:37.234993+0000","last_fullsized":"2023-10-09T04:42:20.912041+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:18.777639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"223'2593","reported_seq":2606,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:43:00.265989+0000","last_change":"2023-10-09T04:42:20.930068+0000","last_active":"2023-10-09T04:43:00.265989+0000","last_peered":"2023-10-09T04:43:00.265989+0000","last_clean":"2023-10-09T04:43:00.265989+0000","last_became_active":"2023-10-09T04:42:20.929778+0000","last_became_peered":"2023-10-09T04:42:20.929778+0000","last_unstale":"2023-10-09T04:43:00.265989+0000","last_undegraded":"2023-10-09T04:43:00.265989+0000","last_fullsized":"2023-10-09T04:43:00.265989+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2593,"log_dups_size":0,"ondisk_log_size":2593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169934848,"num_objects":2593,"num_object_clones":0,"num_object_copies":7779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2593,"num_write_kb":165952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1160,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.184539+0000","last_change":"2023-10-09T04:42:20.921292+0000","last_active":"2023-10-09T04:42:37.184539+0000","last_peered":"2023-10-09T04:42:37.184539+0000","last_clean":"2023-10-09T04:13:51.741721+0000","last_became_active":"2023-10-09T04:42:20.921292+0000","last_became_peered":"2023-10-09T04:42:20.921292+0000","last_unstale":"2023-10-09T04:42:37.184539+0000","last_undegraded":"2023-10-09T04:42:37.184539+0000","last_fullsized":"2023-10-09T04:42:20.917810+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:05.026851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":197,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-10-09T04:42:40.925158+0000","last_change":"2023-10-09T04:42:20.923267+0000","last_active":"2023-10-09T04:42:40.925158+0000","last_peered":"2023-10-09T04:42:40.925158+0000","last_clean":"2023-10-09T04:42:04.812606+0000","last_became_active":"2023-10-09T04:42:20.923267+0000","last_became_peered":"2023-10-09T04:42:20.923267+0000","last_unstale":"2023-10-09T04:42:40.925158+0000","last_undegraded":"2023-10-09T04:42:40.925158+0000","last_fullsized":"2023-10-09T04:42:20.913258+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:17.906710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.234439+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:42:37.234439+0000","last_peered":"2023-10-09T04:42:37.234439+0000","last_clean":"2023-10-09T04:42:37.234439+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:42:37.234439+0000","last_undegraded":"2023-10-09T04:42:37.234439+0000","last_fullsized":"2023-10-09T04:42:37.234439+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"223'2373","reported_seq":2386,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:58.231032+0000","last_change":"2023-10-09T04:42:20.930527+0000","last_active":"2023-10-09T04:42:58.231032+0000","last_peered":"2023-10-09T04:42:58.231032+0000","last_clean":"2023-10-09T04:42:58.231032+0000","last_became_active":"2023-10-09T04:42:20.930219+0000","last_became_peered":"2023-10-09T04:42:20.930219+0000","last_unstale":"2023-10-09T04:42:58.231032+0000","last_undegraded":"2023-10-09T04:42:58.231032+0000","last_fullsized":"2023-10-09T04:42:58.231032+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2373,"log_dups_size":0,"ondisk_log_size":2373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155516928,"num_objects":2373,"num_object_clones":0,"num_object_copies":7119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2373,"num_write_kb":151872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":739,"reported_epoch":222,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:36.386306+0000","last_change":"2023-10-09T04:42:20.935095+0000","last_active":"2023-10-09T04:42:36.386306+0000","last_peered":"2023-10-09T04:42:36.386306+0000","last_clean":"2023-10-09T04:42:04.815275+0000","last_became_active":"2023-10-09T04:42:20.935095+0000","last_became_peered":"2023-10-09T04:42:20.935095+0000","last_unstale":"2023-10-09T04:42:36.386306+0000","last_undegraded":"2023-10-09T04:42:20.928396+0000","last_fullsized":"2023-10-09T04:42:20.928249+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":444,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.259817+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:42:37.259817+0000","last_peered":"2023-10-09T04:42:37.259817+0000","last_clean":"2023-10-09T04:42:37.259817+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:42:37.259817+0000","last_undegraded":"2023-10-09T04:42:37.259817+0000","last_fullsized":"2023-10-09T04:42:37.259817+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":141,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385830+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:42:36.385830+0000","last_peered":"2023-10-09T04:42:36.385830+0000","last_clean":"2023-10-09T04:42:36.385830+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:42:36.385830+0000","last_undegraded":"2023-10-09T04:42:36.385830+0000","last_fullsized":"2023-10-09T04:42:36.385830+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":157,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.164881+0000","last_change":"2023-10-09T04:40:47.825563+0000","last_active":"2023-10-09T04:42:37.164881+0000","last_peered":"2023-10-09T04:42:37.164881+0000","last_clean":"2023-10-09T04:42:37.164881+0000","last_became_active":"2023-10-09T04:40:47.824020+0000","last_became_peered":"2023-10-09T04:40:47.824020+0000","last_unstale":"2023-10-09T04:42:37.164881+0000","last_undegraded":"2023-10-09T04:42:37.164881+0000","last_fullsized":"2023-10-09T04:42:37.164881+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"223'2577","reported_seq":2588,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:59.056169+0000","last_change":"2023-10-09T04:42:39.720459+0000","last_active":"2023-10-09T04:42:59.056169+0000","last_peered":"2023-10-09T04:42:59.056169+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.932318+0000","last_became_peered":"2023-10-09T04:42:20.932318+0000","last_unstale":"2023-10-09T04:42:59.056169+0000","last_undegraded":"2023-10-09T04:42:39.719871+0000","last_fullsized":"2023-10-09T04:42:20.913949+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2577,"log_dups_size":0,"ondisk_log_size":2577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:00.294032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168886272,"num_objects":2577,"num_object_clones":0,"num_object_copies":7731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2577,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2577,"num_write_kb":164928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":2577}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":610,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.259611+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:42:37.259611+0000","last_peered":"2023-10-09T04:42:37.259611+0000","last_clean":"2023-10-09T04:42:37.259611+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:42:37.259611+0000","last_undegraded":"2023-10-09T04:42:37.259611+0000","last_fullsized":"2023-10-09T04:42:37.259611+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":217,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185374+0000","last_change":"2023-10-09T04:38:49.685214+0000","last_active":"2023-10-09T04:42:37.185374+0000","last_peered":"2023-10-09T04:42:37.185374+0000","last_clean":"2023-10-09T04:42:37.185374+0000","last_became_active":"2023-10-09T04:38:49.684807+0000","last_became_peered":"2023-10-09T04:38:49.684807+0000","last_unstale":"2023-10-09T04:42:37.185374+0000","last_undegraded":"2023-10-09T04:42:37.185374+0000","last_fullsized":"2023-10-09T04:42:37.185374+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:01.414710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":200,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.185879+0000","last_change":"2023-10-09T04:42:20.923766+0000","last_active":"2023-10-09T04:42:37.185879+0000","last_peered":"2023-10-09T04:42:37.185879+0000","last_clean":"2023-10-09T04:42:04.818538+0000","last_became_active":"2023-10-09T04:42:20.923766+0000","last_became_peered":"2023-10-09T04:42:20.923766+0000","last_unstale":"2023-10-09T04:42:37.185879+0000","last_undegraded":"2023-10-09T04:42:37.185879+0000","last_fullsized":"2023-10-09T04:42:20.919755+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:45.113343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"223'2598","reported_seq":2608,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:59.073347+0000","last_change":"2023-10-09T04:42:39.732983+0000","last_active":"2023-10-09T04:42:59.073347+0000","last_peered":"2023-10-09T04:42:59.073347+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.917320+0000","last_became_peered":"2023-10-09T04:42:20.917320+0000","last_unstale":"2023-10-09T04:42:59.073347+0000","last_undegraded":"2023-10-09T04:42:39.723997+0000","last_fullsized":"2023-10-09T04:42:20.912241+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2598,"log_dups_size":0,"ondisk_log_size":2598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:35.276341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170262528,"num_objects":2598,"num_object_clones":0,"num_object_copies":7794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2598,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2598,"num_write_kb":166272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":2598}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":440,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260694+0000","last_change":"2023-10-09T04:40:48.818879+0000","last_active":"2023-10-09T04:42:37.260694+0000","last_peered":"2023-10-09T04:42:37.260694+0000","last_clean":"2023-10-09T04:42:37.260694+0000","last_became_active":"2023-10-09T04:40:48.818073+0000","last_became_peered":"2023-10-09T04:40:48.818073+0000","last_unstale":"2023-10-09T04:42:37.260694+0000","last_undegraded":"2023-10-09T04:42:37.260694+0000","last_fullsized":"2023-10-09T04:42:37.260694+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":196,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.234648+0000","last_change":"2023-10-09T04:42:20.933272+0000","last_active":"2023-10-09T04:42:37.234648+0000","last_peered":"2023-10-09T04:42:37.234648+0000","last_clean":"2023-10-09T04:42:04.813045+0000","last_became_active":"2023-10-09T04:42:20.933272+0000","last_became_peered":"2023-10-09T04:42:20.933272+0000","last_unstale":"2023-10-09T04:42:37.234648+0000","last_undegraded":"2023-10-09T04:42:37.234648+0000","last_fullsized":"2023-10-09T04:42:20.912148+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:25.981192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":197,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185464+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:42:37.185464+0000","last_peered":"2023-10-09T04:42:37.185464+0000","last_clean":"2023-10-09T04:42:37.185464+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:42:37.185464+0000","last_undegraded":"2023-10-09T04:42:37.185464+0000","last_fullsized":"2023-10-09T04:42:37.185464+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"223'2393","reported_seq":2407,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:42:58.416501+0000","last_change":"2023-10-09T04:42:20.927101+0000","last_active":"2023-10-09T04:42:58.416501+0000","last_peered":"2023-10-09T04:42:58.416501+0000","last_clean":"2023-10-09T04:42:58.416501+0000","last_became_active":"2023-10-09T04:42:20.926805+0000","last_became_peered":"2023-10-09T04:42:20.926805+0000","last_unstale":"2023-10-09T04:42:58.416501+0000","last_undegraded":"2023-10-09T04:42:58.416501+0000","last_fullsized":"2023-10-09T04:42:58.416501+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2393,"log_dups_size":0,"ondisk_log_size":2393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:24.718363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2393,"num_write_kb":153152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":42,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.260981+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:42:37.260981+0000","last_peered":"2023-10-09T04:42:37.260981+0000","last_clean":"2023-10-09T04:42:37.260981+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:42:37.260981+0000","last_undegraded":"2023-10-09T04:42:37.260981+0000","last_fullsized":"2023-10-09T04:42:37.260981+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":466,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.235773+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:42:37.235773+0000","last_peered":"2023-10-09T04:42:37.235773+0000","last_clean":"2023-10-09T04:42:37.235773+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:42:37.235773+0000","last_undegraded":"2023-10-09T04:42:37.235773+0000","last_fullsized":"2023-10-09T04:42:37.235773+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.259493+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:42:37.259493+0000","last_peered":"2023-10-09T04:42:37.259493+0000","last_clean":"2023-10-09T04:42:37.259493+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:42:37.259493+0000","last_undegraded":"2023-10-09T04:42:37.259493+0000","last_fullsized":"2023-10-09T04:42:37.259493+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"223'2735","reported_seq":2745,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:43:00.089526+0000","last_change":"2023-10-09T04:42:39.729010+0000","last_active":"2023-10-09T04:43:00.089526+0000","last_peered":"2023-10-09T04:43:00.089526+0000","last_clean":"2023-10-09T04:42:19.890843+0000","last_became_active":"2023-10-09T04:42:20.934548+0000","last_became_peered":"2023-10-09T04:42:20.934548+0000","last_unstale":"2023-10-09T04:43:00.089526+0000","last_undegraded":"2023-10-09T04:42:39.725854+0000","last_fullsized":"2023-10-09T04:42:20.928237+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":218,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:42:19.890843+0000","last_clean_scrub_stamp":"2023-10-09T04:42:19.890843+0000","objects_scrubbed":0,"log_size":2735,"log_dups_size":0,"ondisk_log_size":2735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2735,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2735,"num_write_kb":175040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":2735}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":294,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.165258+0000","last_change":"2023-10-09T04:42:20.933140+0000","last_active":"2023-10-09T04:42:37.165258+0000","last_peered":"2023-10-09T04:42:37.165258+0000","last_clean":"2023-10-09T04:42:05.108178+0000","last_became_active":"2023-10-09T04:42:20.933140+0000","last_became_peered":"2023-10-09T04:42:20.933140+0000","last_unstale":"2023-10-09T04:42:37.165258+0000","last_undegraded":"2023-10-09T04:42:37.165258+0000","last_fullsized":"2023-10-09T04:42:20.921841+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":183,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185994+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:42:37.185994+0000","last_peered":"2023-10-09T04:42:37.185994+0000","last_clean":"2023-10-09T04:42:37.185994+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:42:37.185994+0000","last_undegraded":"2023-10-09T04:42:37.185994+0000","last_fullsized":"2023-10-09T04:42:37.185994+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385495+0000","last_change":"2023-10-09T04:38:49.687481+0000","last_active":"2023-10-09T04:42:36.385495+0000","last_peered":"2023-10-09T04:42:36.385495+0000","last_clean":"2023-10-09T04:42:36.385495+0000","last_became_active":"2023-10-09T04:38:49.687039+0000","last_became_peered":"2023-10-09T04:38:49.687039+0000","last_unstale":"2023-10-09T04:42:36.385495+0000","last_undegraded":"2023-10-09T04:42:36.385495+0000","last_fullsized":"2023-10-09T04:42:36.385495+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:37.185519+0000","last_change":"2023-10-09T04:40:47.823824+0000","last_active":"2023-10-09T04:42:37.185519+0000","last_peered":"2023-10-09T04:42:37.185519+0000","last_clean":"2023-10-09T04:42:37.185519+0000","last_became_active":"2023-10-09T04:40:47.817680+0000","last_became_peered":"2023-10-09T04:40:47.817680+0000","last_unstale":"2023-10-09T04:42:37.185519+0000","last_undegraded":"2023-10-09T04:42:37.185519+0000","last_fullsized":"2023-10-09T04:42:37.185519+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:07.850102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:36.385888+0000","last_change":"2023-10-09T04:42:20.934756+0000","last_active":"2023-10-09T04:42:36.385888+0000","last_peered":"2023-10-09T04:42:36.385888+0000","last_clean":"2023-10-09T04:28:19.572818+0000","last_became_active":"2023-10-09T04:42:20.934756+0000","last_became_peered":"2023-10-09T04:42:20.934756+0000","last_unstale":"2023-10-09T04:42:36.385888+0000","last_undegraded":"2023-10-09T04:42:36.385888+0000","last_fullsized":"2023-10-09T04:42:20.930634+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":148,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385762+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:42:36.385762+0000","last_peered":"2023-10-09T04:42:36.385762+0000","last_clean":"2023-10-09T04:42:36.385762+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:42:36.385762+0000","last_undegraded":"2023-10-09T04:42:36.385762+0000","last_fullsized":"2023-10-09T04:42:36.385762+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.234693+0000","last_change":"2023-10-09T04:42:20.917368+0000","last_active":"2023-10-09T04:42:37.234693+0000","last_peered":"2023-10-09T04:42:37.234693+0000","last_clean":"2023-10-09T04:42:04.812851+0000","last_became_active":"2023-10-09T04:42:20.917368+0000","last_became_peered":"2023-10-09T04:42:20.917368+0000","last_unstale":"2023-10-09T04:42:37.234693+0000","last_undegraded":"2023-10-09T04:42:37.234693+0000","last_fullsized":"2023-10-09T04:42:20.912837+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:09.622192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":141,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.386304+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:42:36.386304+0000","last_peered":"2023-10-09T04:42:36.386304+0000","last_clean":"2023-10-09T04:42:36.386304+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:42:36.386304+0000","last_undegraded":"2023-10-09T04:42:36.386304+0000","last_fullsized":"2023-10-09T04:42:36.386304+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.184678+0000","last_change":"2023-10-09T04:42:20.921038+0000","last_active":"2023-10-09T04:42:37.184678+0000","last_peered":"2023-10-09T04:42:37.184678+0000","last_clean":"2023-10-09T04:42:04.818238+0000","last_became_active":"2023-10-09T04:42:20.921038+0000","last_became_peered":"2023-10-09T04:42:20.921038+0000","last_unstale":"2023-10-09T04:42:37.184678+0000","last_undegraded":"2023-10-09T04:42:37.184678+0000","last_fullsized":"2023-10-09T04:42:20.917726+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:25.343316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":115,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.163217+0000","last_change":"2023-10-09T04:42:20.922677+0000","last_active":"2023-10-09T04:42:37.163217+0000","last_peered":"2023-10-09T04:42:37.163217+0000","last_clean":"2023-10-09T04:42:05.108091+0000","last_became_active":"2023-10-09T04:42:20.922677+0000","last_became_peered":"2023-10-09T04:42:20.922677+0000","last_unstale":"2023-10-09T04:42:37.163217+0000","last_undegraded":"2023-10-09T04:42:37.163217+0000","last_fullsized":"2023-10-09T04:42:20.918762+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":222,"state":"active+undersized","last_fresh":"2023-10-09T04:42:37.184498+0000","last_change":"2023-10-09T04:42:20.920794+0000","last_active":"2023-10-09T04:42:37.184498+0000","last_peered":"2023-10-09T04:42:37.184498+0000","last_clean":"2023-10-09T04:22:50.688704+0000","last_became_active":"2023-10-09T04:42:20.920794+0000","last_became_peered":"2023-10-09T04:42:20.920794+0000","last_unstale":"2023-10-09T04:42:37.184498+0000","last_undegraded":"2023-10-09T04:42:37.184498+0000","last_fullsized":"2023-10-09T04:42:20.917408+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:46.291720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":204,"reported_epoch":222,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:42:37.184707+0000","last_change":"2023-10-09T04:42:20.936233+0000","last_active":"2023-10-09T04:42:37.184707+0000","last_peered":"2023-10-09T04:42:37.184707+0000","last_clean":"2023-10-09T04:42:04.818686+0000","last_became_active":"2023-10-09T04:42:20.936233+0000","last_became_peered":"2023-10-09T04:42:20.936233+0000","last_unstale":"2023-10-09T04:42:37.184707+0000","last_undegraded":"2023-10-09T04:42:20.919491+0000","last_fullsized":"2023-10-09T04:42:20.919424+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:01.775406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:42:36.385132+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:42:36.385132+0000","last_peered":"2023-10-09T04:42:36.385132+0000","last_clean":"2023-10-09T04:42:36.385132+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:42:36.385132+0000","last_undegraded":"2023-10-09T04:42:36.385132+0000","last_fullsized":"2023-10-09T04:42:36.385132+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":86,"acting":86,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41128,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":88,"acting":88,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2676228096,"num_objects":40836,"num_object_clones":0,"num_object_copies":122508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15605,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40836,"num_write_kb":2613504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7090798592,"data_stored":7090798592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40836,"ondisk_log_size":40836,"up":42,"acting":42,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430623,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168060,"kb_used_data":862644,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92577092,"statfs":{"total":95995035648,"available":94798942208,"internally_reserved":0,"allocated":883347456,"data_stored":882778269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":3.3540000000000001,"15min":2.6920000000000002},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":4.532,"5min":104.16200000000001,"15min":250.65700000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.93700000000000006,"5min":3.1549999999999998,"15min":3.0550000000000002},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.23400000000000001},"max":{"1min":3.117,"5min":104.129,"15min":251.214},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.304,"15min":1.1950000000000001},"min":{"1min":0.33600000000000002,"5min":0.28899999999999998,"15min":0.28799999999999998},"max":{"1min":1.0700000000000001,"5min":7.3040000000000003,"15min":7.3040000000000003},"last":0.311},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.131,"15min":1.4670000000000001},"min":{"1min":0.502,"5min":0.32600000000000001,"15min":0.28299999999999997},"max":{"1min":3.5329999999999999,"5min":5.9180000000000001,"15min":11.988},"last":0.54600000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.427,"15min":1.855},"min":{"1min":0.38400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8600000000000003,"5min":4.8600000000000003,"15min":106.456},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.821,"5min":1.5229999999999999,"15min":1.8560000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.32400000000000001},"max":{"1min":8.9350000000000005,"5min":8.9350000000000005,"15min":106.708},"last":0.81599999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.5249999999999999,"15min":1.48},"min":{"1min":0.42099999999999999,"5min":0.34000000000000002,"15min":0.318},"max":{"1min":1.22,"5min":8.218,"15min":9.6869999999999994},"last":2.718},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.409,"15min":1.373},"min":{"1min":0.41099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.1439999999999999,"5min":8.2300000000000004,"15min":9.6609999999999996},"last":0.33900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.86499999999999999,"15min":1.2430000000000001},"min":{"1min":0.36499999999999999,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":0.98199999999999998,"5min":4.7939999999999996,"15min":11.683999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.93600000000000005,"15min":1.147},"min":{"1min":0.47699999999999998,"5min":0.36899999999999999,"15min":0.317},"max":{"1min":1.075,"5min":4.2400000000000002,"15min":10.696999999999999},"last":2.8559999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:42:56 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1,"15min":1.621},"min":{"1min":0.48499999999999999,"5min":0.36099999999999999,"15min":0.35699999999999998},"max":{"1min":4.5700000000000003,"5min":5.2610000000000001,"15min":8.0790000000000006},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.157,"5min":1.016,"15min":1.8240000000000001},"min":{"1min":0.48199999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.625,"5min":3.798,"15min":10.786},"last":2.669}]},{"osd":6,"last update":"Mon Oct 9 04:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":1.224,"15min":1.573},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1839999999999999,"5min":6.242,"15min":11.955},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.3799999999999999,"5min":1.1319999999999999,"15min":1.165},"min":{"1min":0.55500000000000005,"5min":0.38300000000000001,"15min":0.35599999999999998},"max":{"1min":5.7149999999999999,"5min":5.7149999999999999,"15min":5.7149999999999999},"last":0.49099999999999999}]}]},{"osd":6,"up_from":146,"seq":627065225323,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118708,"kb_used_data":784556,"kb_used_omap":5,"kb_used_meta":334138,"kb_avail":92626444,"statfs":{"total":95995035648,"available":94849478656,"internally_reserved":0,"allocated":803385344,"data_stored":802824222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":342157920},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.609,"15min":2.6960000000000002},"min":{"1min":0.44900000000000001,"5min":0.25900000000000001,"15min":0.248},"max":{"1min":5.117,"5min":115.63200000000001,"15min":125.666},"last":0.379},{"interface":"front","average":{"1min":1.151,"5min":2.423,"15min":2.6299999999999999},"min":{"1min":0.54100000000000004,"5min":0.29799999999999999,"15min":0.26700000000000002},"max":{"1min":3.8279999999999998,"5min":114.852,"15min":124.991},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.653,"15min":3.246},"min":{"1min":0.52600000000000002,"5min":0.34999999999999998,"15min":0.29199999999999998},"max":{"1min":1.147,"5min":25.765000000000001,"15min":68.155000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6120000000000001,"15min":3.6850000000000001},"min":{"1min":0.46700000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.319,"5min":25.504000000000001,"15min":67.308999999999997},"last":2.008}]},{"osd":2,"last update":"Mon Oct 9 04:41:30 2023","interfaces":[{"interface":"back","average":{"1min":4.9329999999999998,"5min":2.0510000000000002,"15min":4.1589999999999998},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":61.337000000000003,"5min":61.337000000000003,"15min":68.316000000000003},"last":0.84699999999999998},{"interface":"front","average":{"1min":5.9880000000000004,"5min":2.3239999999999998,"15min":4.149},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.313},"max":{"1min":61.649999999999999,"5min":61.649999999999999,"15min":68.210999999999999},"last":1.196}]},{"osd":3,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.5529999999999999,"15min":1.925},"min":{"1min":0.51100000000000001,"5min":0.32100000000000001,"15min":0.31900000000000001},"max":{"1min":9.8810000000000002,"5min":9.8810000000000002,"15min":11.673},"last":0.434},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.5700000000000001,"15min":2.1400000000000001},"min":{"1min":0.53900000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.6349999999999998,"5min":8.3610000000000007,"15min":15.824},"last":2.0459999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.87,"15min":1.355},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.103,"5min":4.5039999999999996,"15min":8.0500000000000007},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.997,"5min":1.163,"15min":1.3939999999999999},"min":{"1min":0.65100000000000002,"5min":0.374,"15min":0.35899999999999999},"max":{"1min":1.5840000000000001,"5min":6.71,"15min":10.324},"last":0.623}]},{"osd":5,"last update":"Mon Oct 9 04:42:52 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.98299999999999998,"15min":1.9139999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.23,"5min":6.3339999999999996,"15min":10.193},"last":1.127},{"interface":"front","average":{"1min":1.171,"5min":1.232,"15min":1.9419999999999999},"min":{"1min":0.58499999999999996,"5min":0.435,"15min":0.38600000000000001},"max":{"1min":3.0249999999999999,"5min":6.6440000000000001,"15min":11.727},"last":4.5199999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.2350000000000001,"15min":1.7709999999999999},"min":{"1min":0.52600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.1519999999999999,"5min":8.2479999999999993,"15min":13.191000000000001},"last":1.9430000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.1890000000000001,"15min":1.548},"min":{"1min":0.38,"5min":0.38,"15min":0.35499999999999998},"max":{"1min":1.3720000000000001,"5min":7.1020000000000003,"15min":8.1180000000000003},"last":1.1599999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958772,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1545200,"kb_used_data":1100712,"kb_used_omap":6,"kb_used_meta":444473,"kb_avail":92199952,"statfs":{"total":95995035648,"available":94412750848,"internally_reserved":0,"allocated":1127129088,"data_stored":1126222365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":455140364},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":4.4119999999999999,"15min":3.1789999999999998},"min":{"1min":0.23999999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":2.0710000000000002,"5min":91.025999999999996,"15min":158.435},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.2390000000000001,"5min":4.6879999999999997,"15min":3.3759999999999999},"min":{"1min":0.26300000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.0350000000000001,"5min":91.536000000000001,"15min":159.38999999999999},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:42:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.403,"15min":2.7829999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":11.789,"5min":90.275999999999996,"15min":157.70400000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":3.5600000000000001,"15min":2.8340000000000001},"min":{"1min":0.29099999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.944,"5min":90.409000000000006,"15min":157.64099999999999},"last":0.70599999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:42:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.9330000000000001,"15min":1.776},"min":{"1min":0.38900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.004,"5min":11.993,"15min":14.369},"last":1.194},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.9059999999999999,"15min":1.752},"min":{"1min":0.316,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.504,"5min":9.4030000000000005,"15min":12.932},"last":1.159}]},{"osd":4,"last update":"Mon Oct 9 04:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2889999999999999,"15min":1.371},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":6.343,"5min":6.343,"15min":12.327},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.4830000000000001,"15min":1.4730000000000001},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0869999999999997,"5min":10.923999999999999,"15min":12.209},"last":0.82499999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.6120000000000001,"15min":1.698},"min":{"1min":0.39400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.782,"5min":6.0810000000000004,"15min":6.9900000000000002},"last":1.6779999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.3420000000000001,"15min":1.6679999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.758,"5min":6.5970000000000004,"15min":7.7460000000000004},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.899,"15min":1.9370000000000001},"min":{"1min":0.51800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.0839999999999996,"5min":7.2560000000000002,"15min":8.9600000000000009},"last":0.622},{"interface":"front","average":{"1min":1.325,"5min":1.9430000000000001,"15min":1.9790000000000001},"min":{"1min":0.58799999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":4.2060000000000004,"5min":7.298,"15min":8.9879999999999995},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:42:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.383,"15min":1.8580000000000001},"min":{"1min":0.46200000000000002,"5min":0.44400000000000001,"15min":0.35899999999999999},"max":{"1min":6.3879999999999999,"5min":6.3879999999999999,"15min":144.613},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.7,"15min":1.9410000000000001},"min":{"1min":0.45200000000000001,"5min":0.42999999999999999,"15min":0.39100000000000001},"max":{"1min":5.1550000000000002,"5min":18.074000000000002,"15min":145.53700000000001},"last":1.2370000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122297,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1404860,"kb_used_data":927412,"kb_used_omap":9,"kb_used_meta":477430,"kb_avail":92340292,"statfs":{"total":95995035648,"available":94556459008,"internally_reserved":0,"allocated":949669888,"data_stored":948754858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":488888817},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:42:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":3.2690000000000001,"15min":2.2759999999999998},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":1.0289999999999999,"5min":223.98500000000001,"15min":223.98500000000001},"last":1.1279999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":3.7130000000000001,"15min":2.3639999999999999},"min":{"1min":0.254,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":4.9210000000000003,"5min":223.13,"15min":223.13},"last":1.304}]},{"osd":2,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":2.145,"15min":2.0510000000000002},"min":{"1min":0.30199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.0370000000000008,"5min":12.166,"15min":61.634999999999998},"last":0.68000000000000005},{"interface":"front","average":{"1min":2.294,"5min":1.905,"15min":1.7769999999999999},"min":{"1min":0.34499999999999997,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":8.9719999999999995,"5min":12.132,"15min":64.677000000000007},"last":0.70499999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:42:37 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.9399999999999999,"15min":1.6479999999999999},"min":{"1min":0.38500000000000001,"5min":0.23400000000000001,"15min":0.20300000000000001},"max":{"1min":1.4770000000000001,"5min":18.303999999999998,"15min":64.622},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.145,"15min":1.863},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.218},"max":{"1min":1.476,"5min":14.956,"15min":64.540999999999997},"last":0.30099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":1.4390000000000001,"15min":1.667},"min":{"1min":0.373,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":7.4080000000000004,"5min":7.4080000000000004,"15min":61.549999999999997},"last":3.3069999999999999},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.2230000000000001,"15min":1.381},"min":{"1min":0.47699999999999998,"5min":0.317,"15min":0.308},"max":{"1min":3.944,"5min":10.218999999999999,"15min":61.902999999999999},"last":1.524}]},{"osd":5,"last update":"Mon Oct 9 04:42:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.7410000000000001,"15min":2.8769999999999998},"min":{"1min":0.58899999999999997,"5min":0.311,"15min":0.311},"max":{"1min":1.5580000000000001,"5min":13.202,"15min":61.972999999999999},"last":1.474},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.4039999999999999,"15min":3.1840000000000002},"min":{"1min":0.59199999999999997,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.5629999999999997,"5min":10.244,"15min":62.417000000000002},"last":1.7410000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.617,"15min":3.0550000000000002},"min":{"1min":0.52600000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.1029999999999998,"5min":10.345000000000001,"15min":62.462000000000003},"last":1.458},{"interface":"front","average":{"1min":1.048,"5min":1.8700000000000001,"15min":3.3010000000000002},"min":{"1min":0.41699999999999998,"5min":0.34699999999999998,"15min":0.32100000000000001},"max":{"1min":3.3029999999999999,"5min":11.57,"15min":61.808},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":1.7669999999999999,"15min":1.9059999999999999},"min":{"1min":0.46200000000000002,"5min":0.39300000000000002,"15min":0.33900000000000002},"max":{"1min":11.643000000000001,"5min":11.643000000000001,"15min":62.006},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.1960000000000002,"5min":1.5880000000000001,"15min":1.8879999999999999},"min":{"1min":0.43099999999999999,"5min":0.39600000000000002,"15min":0.32300000000000001},"max":{"1min":7.5309999999999997,"5min":7.5309999999999997,"15min":61.606999999999999},"last":0.41799999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795238,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402340,"kb_used_data":6428,"kb_used_omap":11,"kb_used_meta":395892,"kb_avail":93342812,"statfs":{"total":95995035648,"available":95583039488,"internally_reserved":0,"allocated":6582272,"data_stored":5702661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12246,"internal_metadata":405393450},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.7509999999999999,"15min":1.4990000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.191},"max":{"1min":10.206,"5min":10.959,"15min":18.155999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.202,"5min":1.6499999999999999,"15min":1.5649999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.20599999999999999},"max":{"1min":6.0449999999999999,"5min":19.219000000000001,"15min":19.219000000000001},"last":0.56899999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:42:03 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":3.3540000000000001,"15min":2.0070000000000001},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":15.227,"5min":189.14099999999999,"15min":189.14099999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.5,"5min":3.198,"15min":2.0739999999999998},"min":{"1min":0.248,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":15.432,"5min":190.023,"15min":190.023},"last":0.497}]},{"osd":3,"last update":"Mon Oct 9 04:41:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6280000000000001,"5min":1.8620000000000001,"15min":1.6140000000000001},"min":{"1min":0.316,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":18.922999999999998,"5min":18.922999999999998,"15min":18.922999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.254,"5min":1.6970000000000001,"15min":1.6339999999999999},"min":{"1min":0.28299999999999997,"5min":0.26600000000000001,"15min":0.23899999999999999},"max":{"1min":18.984000000000002,"5min":18.984000000000002,"15min":18.984000000000002},"last":0.626}]},{"osd":4,"last update":"Mon Oct 9 04:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.006,"15min":1.3620000000000001},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.29599999999999999},"max":{"1min":1.292,"5min":4.6310000000000002,"15min":13.689},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.054,"5min":1.782,"15min":1.7609999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.4009999999999998,"5min":10.002000000000001,"15min":13.759},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:41:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.8160000000000001,"15min":2.7000000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.6499999999999999,"5min":10.057,"15min":14.564},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.647,"5min":1.343,"15min":1.839},"min":{"1min":0.36899999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6870000000000003,"5min":6.6870000000000003,"15min":14.305999999999999},"last":0.68400000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0249999999999999,"5min":1.546,"15min":1.6439999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":6.7320000000000002,"5min":7.2750000000000004,"15min":9.5719999999999992},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.3300000000000001,"15min":1.4350000000000001},"min":{"1min":0.38,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.8300000000000001,"5min":6.9820000000000002,"15min":14.683999999999999},"last":0.72699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:42:03 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.5129999999999999,"15min":1.526},"min":{"1min":0.46800000000000003,"5min":0.44500000000000001,"15min":0.35099999999999998},"max":{"1min":6.7770000000000001,"5min":6.7770000000000001,"15min":14.372999999999999},"last":0.441},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.2969999999999999,"15min":1.498},"min":{"1min":0.51200000000000001,"5min":0.41799999999999998,"15min":0.29899999999999999},"max":{"1min":4.7919999999999998,"5min":6.218,"15min":14.6},"last":0.88200000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631718,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1453172,"kb_used_data":970284,"kb_used_omap":5,"kb_used_meta":482874,"kb_avail":92291980,"statfs":{"total":95995035648,"available":94506987520,"internally_reserved":0,"allocated":993570816,"data_stored":992661588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":494463150},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":5.8159999999999998,"15min":3.109},"min":{"1min":0.22500000000000001,"5min":0.20200000000000001,"15min":0.154},"max":{"1min":9.4619999999999997,"5min":341.28699999999998,"15min":341.28699999999998},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":5.5270000000000001,"15min":3.069},"min":{"1min":0.34100000000000003,"5min":0.22700000000000001,"15min":0.221},"max":{"1min":1.5329999999999999,"5min":341.57799999999997,"15min":341.57799999999997},"last":0.248}]},{"osd":1,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":2.3740000000000001,"15min":1.7989999999999999},"min":{"1min":0.33400000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":1.5169999999999999,"5min":47.637999999999998,"15min":61.395000000000003},"last":0.22500000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":2.6219999999999999,"15min":1.833},"min":{"1min":0.35599999999999998,"5min":0.188,"15min":0.188},"max":{"1min":6.5350000000000001,"5min":47.616999999999997,"15min":61.179000000000002},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:41:32 2023","interfaces":[{"interface":"back","average":{"1min":9.8670000000000009,"5min":3.8799999999999999,"15min":3.109},"min":{"1min":0.30599999999999999,"5min":0.26800000000000002,"15min":0.25},"max":{"1min":74.224999999999994,"5min":74.224999999999994,"15min":181.959},"last":0.433},{"interface":"front","average":{"1min":9.8149999999999995,"5min":4.1440000000000001,"15min":3.0880000000000001},"min":{"1min":0.42899999999999999,"5min":0.24099999999999999,"15min":0.23799999999999999},"max":{"1min":74.167000000000002,"5min":74.167000000000002,"15min":181.91200000000001},"last":0.63400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.603,"15min":1.2589999999999999},"min":{"1min":0.47699999999999998,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":6.7800000000000002,"5min":12.981,"15min":12.981},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.4019999999999999,"15min":1.298},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28000000000000003},"max":{"1min":1.4830000000000001,"5min":6.798,"15min":7.9260000000000002},"last":0.29199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":1.6080000000000001,"15min":1.681},"min":{"1min":0.38600000000000001,"5min":0.33700000000000002,"15min":0.32300000000000001},"max":{"1min":2.3889999999999998,"5min":6.7750000000000004,"15min":8.0109999999999992},"last":1.169},{"interface":"front","average":{"1min":0.72699999999999998,"5min":1.4219999999999999,"15min":2.0539999999999998},"min":{"1min":0.34499999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2649999999999999,"5min":6.8220000000000001,"15min":7.976},"last":0.34200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:42:28 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.6890000000000001,"15min":2.1579999999999999},"min":{"1min":0.44700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.8049999999999999,"5min":8.8119999999999994,"15min":11.842000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.633,"15min":1.7310000000000001},"min":{"1min":0.45300000000000001,"5min":0.40400000000000003,"15min":0.35399999999999998},"max":{"1min":6.6970000000000001,"5min":13.004,"15min":13.004},"last":0.69799999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:42:33 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5589999999999999,"15min":2.3540000000000001},"min":{"1min":0.499,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.498,"5min":8.1430000000000007,"15min":198.012},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.7190000000000001,"15min":2.3039999999999998},"min":{"1min":0.58099999999999996,"5min":0.38,"15min":0.34599999999999997},"max":{"1min":6.7519999999999998,"5min":13.028,"15min":198.05199999999999},"last":0.41599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019320,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1415732,"kb_used_data":1114412,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92329420,"statfs":{"total":95995035648,"available":94545326080,"internally_reserved":0,"allocated":1141157888,"data_stored":1140596279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.5189999999999999,"15min":1.5860000000000001},"min":{"1min":0.33700000000000002,"5min":0.28599999999999998,"15min":0.24099999999999999},"max":{"1min":4.4900000000000002,"5min":22.841000000000001,"15min":48.450000000000003},"last":2.3610000000000002},{"interface":"front","average":{"1min":1.746,"5min":2.0790000000000002,"15min":1.929},"min":{"1min":0.36099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.414999999999999,"5min":22.966000000000001,"15min":49.283000000000001},"last":2.3849999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:42:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.4179999999999999,"15min":1.679},"min":{"1min":0.437,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":3.5670000000000002,"5min":13.984999999999999,"15min":79.361999999999995},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.028,"5min":1.206,"15min":1.7190000000000001},"min":{"1min":0.34100000000000003,"5min":0.33900000000000002,"15min":0.29199999999999998},"max":{"1min":5.3019999999999996,"5min":5.3929999999999998,"15min":79.292000000000002},"last":2.9289999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.28,"15min":1.857},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.31},"max":{"1min":4.6989999999999998,"5min":7.5730000000000004,"15min":77.537000000000006},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.5510000000000002,"5min":1.595,"15min":2.0139999999999998},"min":{"1min":0.39500000000000002,"5min":0.374,"15min":0.311},"max":{"1min":26.292999999999999,"5min":26.292999999999999,"15min":77.188000000000002},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.3819999999999999,"15min":1.3400000000000001},"min":{"1min":0.41899999999999998,"5min":0.371,"15min":0.317},"max":{"1min":1.2709999999999999,"5min":9.5410000000000004,"15min":10.417},"last":2.3439999999999999},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.2190000000000001,"15min":1.3700000000000001},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":1.2410000000000001,"5min":9.484,"15min":11.112},"last":0.999}]},{"osd":5,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.82299999999999995,"15min":1.018},"min":{"1min":0.46600000000000003,"5min":0.312,"15min":0.312},"max":{"1min":1.8560000000000001,"5min":4.8399999999999999,"15min":10.412000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.78900000000000003,"15min":1.389},"min":{"1min":0.35899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.252,"5min":4.0830000000000002,"15min":6.4470000000000001},"last":2.593}]},{"osd":6,"last update":"Mon Oct 9 04:42:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0369999999999999,"15min":0.96299999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":4.2050000000000001,"5min":5.0129999999999999,"15min":5.0129999999999999},"last":3.2570000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.92900000000000005,"15min":1.171},"min":{"1min":0.45900000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.3940000000000001,"5min":4.6459999999999999,"15min":10.257999999999999},"last":3.3769999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:42:25 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.0109999999999999,"15min":1.1879999999999999},"min":{"1min":0.57699999999999996,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":6.6849999999999996,"5min":9.0370000000000008,"15min":11.725},"last":2.3929999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.105,"15min":1.2310000000000001},"min":{"1min":0.49199999999999999,"5min":0.42899999999999999,"15min":0.30399999999999999},"max":{"1min":4.1769999999999996,"5min":7.5209999999999999,"15min":13.349},"last":2.4129999999999998}]}]},{"osd":5,"up_from":150,"seq":644245094502,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1659428,"kb_used_data":1322012,"kb_used_omap":6,"kb_used_meta":337401,"kb_avail":92085724,"statfs":{"total":95995035648,"available":94295781376,"internally_reserved":0,"allocated":1353740288,"data_stored":1353190511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":345498899},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.4019999999999999,"15min":1.6479999999999999},"min":{"1min":0.50800000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.0629999999999997,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":4.0629999999999997},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.6100000000000001,"15min":1.8680000000000001},"min":{"1min":0.56100000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.8359999999999999,"5min":7.3200000000000003,"15min":8.2319999999999993},"last":2.3199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.4059999999999999,"15min":1.871},"min":{"1min":0.33200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.1739999999999999,"5min":6.6349999999999998,"15min":7.2750000000000004},"last":2.1739999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"min":{"1min":0.65400000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.468,"5min":10.678000000000001,"15min":10.678000000000001},"last":7.468}]},{"osd":2,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":2.0419999999999998,"15min":2.3490000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":20.786999999999999,"5min":22.701000000000001,"15min":22.701000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":2.093,"5min":1.7609999999999999,"15min":2.117},"min":{"1min":0.40799999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":17.806999999999999,"5min":19.567,"15min":19.567},"last":1.0940000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.444,"15min":1.704},"min":{"1min":0.37,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.2010000000000001,"5min":6.766,"15min":7.3250000000000002},"last":2.2010000000000001},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.6759999999999999,"15min":2.2629999999999999},"min":{"1min":0.67200000000000004,"5min":0.373,"15min":0.373},"max":{"1min":3.915,"5min":6.9130000000000003,"15min":8.2859999999999996},"last":2.3439999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.095,"15min":1.6399999999999999},"min":{"1min":0.58999999999999997,"5min":0.374,"15min":0.32600000000000001},"max":{"1min":3.7629999999999999,"5min":6.8550000000000004,"15min":8.7300000000000004},"last":2.2850000000000001},{"interface":"front","average":{"1min":1.2290000000000001,"5min":0.94199999999999995,"15min":1.339},"min":{"1min":0.44600000000000001,"5min":0.36499999999999999,"15min":0.32700000000000001},"max":{"1min":3.246,"5min":7.827,"15min":7.827},"last":2.1499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.127,"15min":1.5669999999999999},"min":{"1min":0.42399999999999999,"5min":0.371,"15min":0.36799999999999999},"max":{"1min":3.5550000000000002,"5min":7.8479999999999999,"15min":11.087},"last":2.0779999999999998},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.115,"15min":1.704},"min":{"1min":0.627,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.726,"5min":6.181,"15min":9.3659999999999997},"last":2.226}]},{"osd":7,"last update":"Mon Oct 9 04:42:59 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.2569999999999999,"15min":1.974},"min":{"1min":0.60399999999999998,"5min":0.41499999999999998,"15min":0.38500000000000001},"max":{"1min":3.96,"5min":5.7649999999999997,"15min":39.771000000000001},"last":3.4340000000000002},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.177,"15min":2.379},"min":{"1min":0.47699999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.5819999999999999,"5min":8.1910000000000007,"15min":39.905000000000001},"last":2.1179999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8027,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":960364544,"data_stored":960364544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1163329536,"data_stored":1163329536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1018822656,"data_stored":1018822656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":873332736,"data_stored":873332736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1375338496,"data_stored":1375338496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":806420480,"data_stored":806420480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":893190144,"data_stored":893190144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:43:06.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-10-09T04:43:06.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:06.144 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:43:06.151 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:43:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:06.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.287 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:06.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:06.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:06.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:06.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:06.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:06.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 52219 52203 120.829 91.25 0.0152915 0.00827019 2023-10-09T04:43:06.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:06.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:06.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:06.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:06.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:06.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:06.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:07.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:07.068 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:07.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:07.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:07.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:07.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:07.456 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:07.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:07.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:07.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:07.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 16 53716 53700 119.855 93.5625 0.00928934 0.00833799 2023-10-09T04:43:07.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:07.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:07.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:07.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:07.883 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:07 smithi153 bash[59565]: cluster 2023-10-09T04:43:06.559450+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v912: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 18477/145689 objects degraded (12.682%) 2023-10-09T04:43:07.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:07.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:07.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:07.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:07 smithi082 bash[77063]: cluster 2023-10-09T04:43:06.559450+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v912: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 18477/145689 objects degraded (12.682%) 2023-10-09T04:43:07.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:07 smithi082 bash[79733]: cluster 2023-10-09T04:43:06.559450+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v912: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 18477/145689 objects degraded (12.682%) 2023-10-09T04:43:08.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.037 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:08.117 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:43:08.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:43:08.169 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:08.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:08.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:08.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:08.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:08.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:08.650 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:43:08.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.694 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:08.723 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 55066 55050 118.631 84.375 0.00467229 0.00842365 2023-10-09T04:43:08.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:08.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:08.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:08.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:08.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:09.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:09.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:09.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:09.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:09.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:09.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:09.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:09.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:09.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 56520 56504 117.705 90.875 0.00468571 0.00849013 2023-10-09T04:43:09.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:09.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:09.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:09.903 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:09 smithi153 bash[59565]: cluster 2023-10-09T04:43:08.560005+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v913: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18910/152682 objects degraded (12.385%) 2023-10-09T04:43:09.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:09.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:09.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:09 smithi082 bash[77063]: cluster 2023-10-09T04:43:08.560005+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v913: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18910/152682 objects degraded (12.385%) 2023-10-09T04:43:09.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:09] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-09T04:43:09.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:09 smithi082 bash[79733]: cluster 2023-10-09T04:43:08.560005+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v913: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 18910/152682 objects degraded (12.385%) 2023-10-09T04:43:10.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:10.063 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:10.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:10.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:10.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:10.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:10.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:10.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:10.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:10.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:10.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 57553 57537 115.991 64.5625 0.00902439 0.00861608 2023-10-09T04:43:10.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:10.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:10.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:10.861 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:10 smithi082 bash[77063]: cluster 2023-10-09T04:43:09.906714+0000 mon.a (mon.0) 1289 : cluster 3 Health check update: Degraded data redundancy: 18910/152682 objects degraded (12.385%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:10.862 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:10 smithi082 bash[79733]: cluster 2023-10-09T04:43:09.906714+0000 mon.a (mon.0) 1289 : cluster 3 Health check update: Degraded data redundancy: 18910/152682 objects degraded (12.385%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:10.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:10.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:10.916 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:10.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:11.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:11.119 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:11.130 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:10 smithi153 bash[59565]: cluster 2023-10-09T04:43:09.906714+0000 mon.a (mon.0) 1289 : cluster 3 Health check update: Degraded data redundancy: 18910/152682 objects degraded (12.385%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:11.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:11.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:11.255 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:11.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:11.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:11.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:11.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:11.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:11.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:11.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 58878 58862 114.954 82.8125 0.00749733 0.00869414 2023-10-09T04:43:11.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:11.819 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:11.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:11.922 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:11 smithi153 bash[59565]: cluster 2023-10-09T04:43:10.560813+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v914: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:11.924 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:11.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:12.057 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:12.083 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:12.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:12.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:11 smithi082 bash[77063]: cluster 2023-10-09T04:43:10.560813+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v914: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:12.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:11 smithi082 bash[79733]: cluster 2023-10-09T04:43:10.560813+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v914: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:12.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:12.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:12.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:12.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:12.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:12.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:12.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:12.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:12.713 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 60266 60250 114.099 86.75 0.00850554 0.0087587 2023-10-09T04:43:12.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:12.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:12.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:12.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:12.951 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:12.952 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:12.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:12.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:13.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:13.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:13.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:13.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:13.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:13.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:13.509 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:13.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:13.634 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:13.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:13.714 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 61559 61543 113.12 80.8125 0.00841659 0.00883502 2023-10-09T04:43:13.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:13.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:13.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:13.927 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:13.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:14.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:14.040 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:13 smithi082 bash[79733]: cluster 2023-10-09T04:43:12.561412+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v915: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:14.041 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:13 smithi082 bash[77063]: cluster 2023-10-09T04:43:12.561412+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v915: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:14.060 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:14.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:14.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:13 smithi153 bash[59565]: cluster 2023-10-09T04:43:12.561412+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v915: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 21220/167418 objects degraded (12.675%) 2023-10-09T04:43:14.201 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:14.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:14.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:14.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:14.534 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:43:14.535 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:43:14.575 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:14.711 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:14.714 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 62838 62822 112.171 79.9375 0.0119227 0.00890821 2023-10-09T04:43:14.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:14.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:14.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:14.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:14.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:15.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:15.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:15.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:15.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:15.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:15.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:15.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:15.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[77063]: audit 2023-10-09T04:43:14.289200+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[77063]: audit 2023-10-09T04:43:14.290556+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[77063]: cluster 2023-10-09T04:43:14.562026+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v916: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 22822/179679 objects degraded (12.702%) 2023-10-09T04:43:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[77063]: cluster 2023-10-09T04:43:14.907955+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 21220/167418 objects degraded (12.675%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[79733]: audit 2023-10-09T04:43:14.289200+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[79733]: audit 2023-10-09T04:43:14.290556+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[79733]: cluster 2023-10-09T04:43:14.562026+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v916: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 22822/179679 objects degraded (12.702%) 2023-10-09T04:43:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:15 smithi082 bash[79733]: cluster 2023-10-09T04:43:14.907955+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 21220/167418 objects degraded (12.675%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:15.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:15.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:15 smithi153 bash[59565]: audit 2023-10-09T04:43:14.289200+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:15.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:15 smithi153 bash[59565]: audit 2023-10-09T04:43:14.290556+0000 mon.a (mon.0) 1291 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:15.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:15 smithi153 bash[59565]: cluster 2023-10-09T04:43:14.562026+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v916: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 22822/179679 objects degraded (12.702%) 2023-10-09T04:43:15.514 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:15 smithi153 bash[59565]: cluster 2023-10-09T04:43:14.907955+0000 mon.a (mon.0) 1292 : cluster 3 Health check update: Degraded data redundancy: 21220/167418 objects degraded (12.675%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:15.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:15.634 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:15.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:15.714 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 64208 64192 111.434 85.625 0.0087835 0.00896901 2023-10-09T04:43:15.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:43:15.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:15.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:15.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:15.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:15.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:16.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:16.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:16.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:16.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:16.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:16.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:16.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:16.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:16.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:16.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:16.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:16.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:16.714 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 65422 65406 110.473 75.875 0.00324127 0.00904631 2023-10-09T04:43:16.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:16.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:16.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:16.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:16.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:16.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:17.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:17.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:17.136 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:17.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:17.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:17.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:17.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:17.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:17.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:17.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:17.638 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:17.647 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:17 smithi153 bash[59565]: cluster 2023-10-09T04:43:16.562827+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v917: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 23615/187062 objects degraded (12.624%) 2023-10-09T04:43:17.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:17.714 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 66411 66395 109.192 61.8125 0.0149872 0.00915323 2023-10-09T04:43:17.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:17.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:17.901 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:17 smithi082 bash[77063]: cluster 2023-10-09T04:43:16.562827+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v917: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 23615/187062 objects degraded (12.624%) 2023-10-09T04:43:17.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:17.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:17.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:18.036 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:17 smithi082 bash[79733]: cluster 2023-10-09T04:43:16.562827+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v917: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 23615/187062 objects degraded (12.624%) 2023-10-09T04:43:18.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:18.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:18.378 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:18.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:18.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:18.517 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:18.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:18.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:18.715 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 67649 67633 108.376 77.375 0.0133729 0.00922282 2023-10-09T04:43:18.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:18.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:18.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:18.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:18.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:19.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:19.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:19.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:19.193 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:19.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:19.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:19.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:19.459 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:19.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:19.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:19.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:19.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:43:19.715189+0000 min lat: 0.00125881 max lat: 0.429926 avg lat: 0.00928755 2023-10-09T04:43:19.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:43:19.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 68892 68876 107.605 77.6875 0.0161205 0.00928755 2023-10-09T04:43:19.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:19.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:19.847 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:19 smithi153 bash[59565]: cluster 2023-10-09T04:43:18.563625+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v918: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 24035/193086 objects degraded (12.448%) 2023-10-09T04:43:19.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:19.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:19.977 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:19 smithi082 bash[77063]: cluster 2023-10-09T04:43:18.563625+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v918: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 24035/193086 objects degraded (12.448%) 2023-10-09T04:43:19.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:19] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T04:43:19.978 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:19 smithi082 bash[79733]: cluster 2023-10-09T04:43:18.563625+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v918: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 24035/193086 objects degraded (12.448%) 2023-10-09T04:43:19.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:19.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:20.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:20.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:20.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:20.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:20.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:20.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:20.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.657 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:20 smithi153 bash[59565]: cluster 2023-10-09T04:43:19.909081+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 24035/193086 objects degraded (12.448%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:20.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:20.715 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 70104 70088 106.828 75.75 0.00354562 0.00935466 2023-10-09T04:43:20.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:20.785 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:20 smithi082 bash[79733]: cluster 2023-10-09T04:43:19.909081+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 24035/193086 objects degraded (12.448%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:20.786 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:20 smithi082 bash[77063]: cluster 2023-10-09T04:43:19.909081+0000 mon.a (mon.0) 1293 : cluster 3 Health check update: Degraded data redundancy: 24035/193086 objects degraded (12.448%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:43:20.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:20.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:20.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:21.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:21.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:21.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:21.174 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:21.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:21.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:21.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:21.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:21.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:21.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:21.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:21.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:21.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:21.705 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:21.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 71256 71240 105.998 72 0.00461588 0.00942968 2023-10-09T04:43:21.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:21.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:21.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:21.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:21.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:21.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:22.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:21 smithi153 bash[59565]: cluster 2023-10-09T04:43:20.564367+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v919: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:22.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:22.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:21 smithi082 bash[77063]: cluster 2023-10-09T04:43:20.564367+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v919: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:22.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:21 smithi082 bash[79733]: cluster 2023-10-09T04:43:20.564367+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v919: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:22.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:22.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:22.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:22.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:22.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:22.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:22.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:22.700 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:43:22.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 72462 72446 105.286 75.375 0.0136007 0.00949131 2023-10-09T04:43:22.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:22.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:22.774 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:43:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:22.813 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:43:22.847 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:43:22.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.889 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:43:22.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:22.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:22.921 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:43:22.968 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:43:22.969 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:43:23.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:23.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:23.142 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:23.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:23.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:22 smithi153 bash[59565]: audit 2023-10-09T04:43:22.638792+0000 mon.a (mon.0) 1294 : audit 0 from='client.? 172.21.15.82:0/723396844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:23.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:22 smithi082 bash[77063]: audit 2023-10-09T04:43:22.638792+0000 mon.a (mon.0) 1294 : audit 0 from='client.? 172.21.15.82:0/723396844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:23.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:22 smithi082 bash[79733]: audit 2023-10-09T04:43:22.638792+0000 mon.a (mon.0) 1294 : audit 0 from='client.? 172.21.15.82:0/723396844' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:23.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:23.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:23.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:23.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:23.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:23.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:23.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:23.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:23.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 73597 73581 104.505 70.9375 0.0102379 0.00956362 2023-10-09T04:43:23.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:23.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:23.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:23.948 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 8 pgp_num to 16 2023-10-09T04:43:23.950 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:43:23.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:23.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:24.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:24.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:23 smithi153 bash[59565]: cluster 2023-10-09T04:43:22.565200+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v920: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:24.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:23 smithi153 bash[59565]: audit 2023-10-09T04:43:23.031898+0000 mon.a (mon.0) 1295 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:43:24.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:23 smithi153 bash[59565]: audit 2023-10-09T04:43:23.084176+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.82:0/3132585352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.067 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:23 smithi153 bash[59565]: audit 2023-10-09T04:43:23.085165+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:24.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:24.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[77063]: cluster 2023-10-09T04:43:22.565200+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v920: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:24.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[77063]: audit 2023-10-09T04:43:23.031898+0000 mon.a (mon.0) 1295 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:43:24.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[77063]: audit 2023-10-09T04:43:23.084176+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.82:0/3132585352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.207 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[77063]: audit 2023-10-09T04:43:23.085165+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.207 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[79733]: cluster 2023-10-09T04:43:22.565200+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v920: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 25871/205230 objects degraded (12.606%) 2023-10-09T04:43:24.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[79733]: audit 2023-10-09T04:43:23.031898+0000 mon.a (mon.0) 1295 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:43:24.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[79733]: audit 2023-10-09T04:43:23.084176+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.82:0/3132585352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.208 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:23 smithi082 bash[79733]: audit 2023-10-09T04:43:23.085165+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:24.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:24.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:24.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:24.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:24.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:24.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:24.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:24.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 74660 74644 103.659 66.4375 0.0104217 0.00964077 2023-10-09T04:43:24.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:24.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:24.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:24.891 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:24.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:24.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:25.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:25.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:23.932136+0000 mon.a (mon.0) 1297 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:23.937382+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:23.938945+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: audit 2023-10-09T04:43:23.942348+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:43:25.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: audit 2023-10-09T04:43:23.942427+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:25.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:23.942491+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e225: 8 total, 7 up, 8 in 2023-10-09T04:43:25.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:23.952361+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:24 smithi153 bash[59565]: cluster 2023-10-09T04:43:24.910294+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 25871/205230 objects degraded (12.606%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:23.932136+0000 mon.a (mon.0) 1297 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:23.937382+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:23.938945+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: audit 2023-10-09T04:43:23.942348+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: audit 2023-10-09T04:43:23.942427+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:23.942491+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e225: 8 total, 7 up, 8 in 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:23.952361+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[77063]: cluster 2023-10-09T04:43:24.910294+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 25871/205230 objects degraded (12.606%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:23.932136+0000 mon.a (mon.0) 1297 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:23.937382+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:23.938945+0000 osd.4 (osd.4) 173 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: audit 2023-10-09T04:43:23.942348+0000 mon.a (mon.0) 1298 : audit 1 from='client.? 172.21.15.82:0/520866006' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: audit 2023-10-09T04:43:23.942427+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:23.942491+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e225: 8 total, 7 up, 8 in 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:23.952361+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T04:43:25.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:24 smithi082 bash[79733]: cluster 2023-10-09T04:43:24.910294+0000 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 25871/205230 objects degraded (12.606%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:25.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:25.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:43:25.314 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:43:25.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:25.524 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:25.554 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:25.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:25.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:25.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 75759 75743 102.899 68.6875 0.0145577 0.00971326 2023-10-09T04:43:25.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:25.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:25.958 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:25.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:26.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:26.070 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:25 smithi153 bash[59565]: cluster 2023-10-09T04:43:24.565903+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v922: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 27300/216306 objects degraded (12.621%) 2023-10-09T04:43:26.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:26.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:26.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:26.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:25 smithi082 bash[77063]: cluster 2023-10-09T04:43:24.565903+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v922: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 27300/216306 objects degraded (12.621%) 2023-10-09T04:43:26.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:25 smithi082 bash[79733]: cluster 2023-10-09T04:43:24.565903+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v922: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 27300/216306 objects degraded (12.621%) 2023-10-09T04:43:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:26.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:26.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:26.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:26.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:26.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:26.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:26.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:26.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:26.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:26.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 16 76780 76764 102.067 63.8125 0.00510179 0.00979165 2023-10-09T04:43:26.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:26.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:26.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:26.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:26.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:26.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:27.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:27.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:27.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:27.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:27.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:43:27.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:27.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:27.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:27.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:27.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:27.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 77937 77921 101.447 72.3125 0.0248131 0.00985117 2023-10-09T04:43:27.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:27.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:27.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:27.958 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:27.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:28.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:28.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:28.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:27 smithi082 bash[77063]: cluster 2023-10-09T04:43:26.566926+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v923: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 28053/222816 objects degraded (12.590%) 2023-10-09T04:43:28.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:27 smithi082 bash[79733]: cluster 2023-10-09T04:43:26.566926+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v923: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 28053/222816 objects degraded (12.590%) 2023-10-09T04:43:28.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:28.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:28.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:28.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:27 smithi153 bash[59565]: cluster 2023-10-09T04:43:26.566926+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v923: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 28053/222816 objects degraded (12.590%) 2023-10-09T04:43:28.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:28.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:28.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:28.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:28.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:28.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:28.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 79068 79052 100.819 70.6875 0.0073468 0.00991332 2023-10-09T04:43:28.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:28.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:28.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:28.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:28.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:29.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.072 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:29.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:29.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:29.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:29.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:29.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:29.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.595 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:29.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:29.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:29.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:29] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T04:43:29.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 80176 80160 100.188 69.25 0.0123777 0.00997507 2023-10-09T04:43:29.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:29.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:29.841 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:43:29.841 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:43:29.841 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.841 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:80301 8.e 8:7062127b:::benchmark_data_smithi082_175677_object80300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:29.826834+0000", 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.012866311, 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012893381000000001, 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 80301 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.842 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826834+0000", 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826834+0000", 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826833+0000", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826857+0000", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4287999999999999e-05 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826869+0000", 2023-10-09T04:43:29.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2320999999999999e-05 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.826878+0000", 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2600000000000005e-06 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.827273+0000", 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00039535299999999998 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.828054+0000", 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00078079699999999996 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 4", 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.828202+0000", 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000148362 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:43:29.846 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.837674+0000", 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0094719739999999993 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:80304 8.2 e225/218 8:4da1c933:::benchmark_data_smithi082_175677_object80303:head v 225'4944, mlcod=225'4941)", 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:29.830472+0000", 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0092289030000000001, 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0093142729999999996, 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:43:29.847 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830472+0000", 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830472+0000", 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830471+0000", 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:43:29.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830507+0000", 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6016000000000002e-05 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830522+0000", 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5369000000000001e-05 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830530+0000", 2023-10-09T04:43:29.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0460000000000006e-06 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830608+0000", 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8081e-05 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.830634+0000", 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5715e-05 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.851 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:80307 8.e 8:77feaedd:::benchmark_data_smithi082_175677_object80306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:29.833022+0000", 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0066786679999999996, 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068152350000000002, 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:43:29.852 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 80307 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833022+0000", 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833022+0000", 2023-10-09T04:43:29.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833020+0000", 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833051+0000", 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0676999999999998e-05 2023-10-09T04:43:29.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833071+0000", 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0208e-05 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833080+0000", 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3139999999999993e-06 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833712+0000", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00063129600000000005 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833919+0000", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020783100000000001 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 4", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.834092+0000", 2023-10-09T04:43:29.856 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00017239299999999999 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:80305 8.2 e225/218 8:42cbacd7:::benchmark_data_smithi082_175677_object80304:head v 225'4945, mlcod=225'4942)", 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:29.833674+0000", 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.006026165, 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0062083329999999999, 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:43:29.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833674+0000", 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833674+0000", 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:43:29.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833674+0000", 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833706+0000", 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1738e-05 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833718+0000", 2023-10-09T04:43:29.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2095e-05 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833725+0000", 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5009999999999998e-06 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833894+0000", 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016883999999999999 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.833991+0000", 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7427000000000001e-05 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:80315 8.e 8:7958810b:::benchmark_data_smithi082_175677_object80314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:29.839410+0000", 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00029075999999999999, 2023-10-09T04:43:29.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0005109, 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 80315 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839410+0000", 2023-10-09T04:43:29.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839410+0000", 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839408+0000", 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839439+0000", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0954999999999998e-05 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839456+0000", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7079999999999999e-05 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839465+0000", 2023-10-09T04:43:29.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4849999999999998e-06 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839509+0000", 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3652000000000002e-05 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839635+0000", 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012604099999999999 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:43:29.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 4", 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:29.839762+0000", 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012706300000000001 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 5 2023-10-09T04:43:29.866 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:43:29.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:29.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:29.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:43:29.969 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:30.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:30.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:30.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[77063]: cluster 2023-10-09T04:43:28.567920+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v924: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 28404/227997 objects degraded (12.458%) 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[77063]: audit 2023-10-09T04:43:29.287156+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[77063]: audit 2023-10-09T04:43:29.290534+0000 mon.a (mon.0) 1303 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[77063]: cluster 2023-10-09T04:43:29.911660+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 28404/227997 objects degraded (12.458%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[79733]: cluster 2023-10-09T04:43:28.567920+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v924: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 28404/227997 objects degraded (12.458%) 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[79733]: audit 2023-10-09T04:43:29.287156+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[79733]: audit 2023-10-09T04:43:29.290534+0000 mon.a (mon.0) 1303 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:30.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:29 smithi082 bash[79733]: cluster 2023-10-09T04:43:29.911660+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 28404/227997 objects degraded (12.458%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:30.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:30.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.412 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:29 smithi153 bash[59565]: cluster 2023-10-09T04:43:28.567920+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v924: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 28404/227997 objects degraded (12.458%) 2023-10-09T04:43:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:29 smithi153 bash[59565]: audit 2023-10-09T04:43:29.287156+0000 mon.a (mon.0) 1302 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:29 smithi153 bash[59565]: audit 2023-10-09T04:43:29.290534+0000 mon.a (mon.0) 1303 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:30.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:29 smithi153 bash[59565]: cluster 2023-10-09T04:43:29.911660+0000 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 28404/227997 objects degraded (12.458%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:30.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:30.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:30.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:30.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:30.716 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 81250 81234 99.5395 67.125 0.0107917 0.0100415 2023-10-09T04:43:30.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:30.903 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:30.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:31.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:31.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:31.202 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:31.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:43:31.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:31.334 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:31.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:31.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:31.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:31.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:31.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:31.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 82332 82316 98.9257 67.625 0.00900767 0.0101029 2023-10-09T04:43:31.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:31.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:31.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:31.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:31.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:31.990 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:31 smithi153 bash[59565]: cluster 2023-10-09T04:43:30.568850+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v925: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:32.027 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:32.131 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:32.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:32.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:32.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:32.295 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:31 smithi082 bash[77063]: cluster 2023-10-09T04:43:30.568850+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v925: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:32.295 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:31 smithi082 bash[79733]: cluster 2023-10-09T04:43:30.568850+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v925: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:32.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:32.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:32.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:32.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:32.583 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:32.687 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:32.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 83421 83405 98.3432 68.0625 0.0121448 0.0101627 2023-10-09T04:43:32.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:32.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:32.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:32.965 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:32.995 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 7, 6, 5] 2023-10-09T04:43:32.995 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:43:32.996 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.982832888051702 2023-10-09T04:43:32.996 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 0.982832888051702 2023-10-09T04:43:32.997 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:32.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:33.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:33.125 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:33.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:33.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:33.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:33.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:33.400 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:33.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:33.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:33.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:33.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:33.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 84455 84439 97.7187 64.625 0.0202333 0.0102283 2023-10-09T04:43:33.787 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:33.817 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:33.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:33.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:33.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:34.047 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: cluster 2023-10-09T04:43:32.569533+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v926: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:34.047 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: audit 2023-10-09T04:43:33.432316+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:34.047 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: audit 2023-10-09T04:43:33.960815+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: audit 2023-10-09T04:43:33.967762+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: audit 2023-10-09T04:43:33.978965+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[77063]: audit 2023-10-09T04:43:33.989494+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: cluster 2023-10-09T04:43:32.569533+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v926: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: audit 2023-10-09T04:43:33.432316+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: audit 2023-10-09T04:43:33.960815+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: audit 2023-10-09T04:43:33.967762+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: audit 2023-10-09T04:43:33.978965+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.048 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:34 smithi082 bash[79733]: audit 2023-10-09T04:43:33.989494+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:34.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:34.168 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:34.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:34.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: cluster 2023-10-09T04:43:32.569533+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v926: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 30137/239238 objects degraded (12.597%) 2023-10-09T04:43:34.190 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: audit 2023-10-09T04:43:33.432316+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:34.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: audit 2023-10-09T04:43:33.960815+0000 mon.a (mon.0) 1306 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: audit 2023-10-09T04:43:33.967762+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: audit 2023-10-09T04:43:33.978965+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.191 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:34 smithi153 bash[59565]: audit 2023-10-09T04:43:33.989494+0000 mon.a (mon.0) 1309 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:34.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:34.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:34.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:34.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:34.458 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:34.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:34.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:34.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:34.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 85404 85388 97.0204 59.3125 0.0108146 0.0103022 2023-10-09T04:43:34.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:34.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:34.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:34.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:34.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:34.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:35.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:35.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:35.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:35.270 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:35 smithi082 bash[79733]: cluster 2023-10-09T04:43:34.913145+0000 mon.a (mon.0) 1310 : cluster 3 Health check update: Degraded data redundancy: 30137/239238 objects degraded (12.597%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:35.270 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:35 smithi082 bash[77063]: cluster 2023-10-09T04:43:34.913145+0000 mon.a (mon.0) 1310 : cluster 3 Health check update: Degraded data redundancy: 30137/239238 objects degraded (12.597%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:35.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:35.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:35.404 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:35 smithi153 bash[59565]: cluster 2023-10-09T04:43:34.913145+0000 mon.a (mon.0) 1310 : cluster 3 Health check update: Degraded data redundancy: 30137/239238 objects degraded (12.597%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:35.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:35.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:35.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:35.666 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.694 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:35.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 86469 86453 96.4763 66.5625 0.0182836 0.0103594 2023-10-09T04:43:35.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.829 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:35.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.961 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:35.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:36.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:36.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:36.212 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:36 smithi153 bash[59565]: cluster 2023-10-09T04:43:34.570448+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v927: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 31406/249201 objects degraded (12.603%) 2023-10-09T04:43:36.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:36 smithi082 bash[77063]: cluster 2023-10-09T04:43:34.570448+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v927: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 31406/249201 objects degraded (12.603%) 2023-10-09T04:43:36.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:36 smithi082 bash[79733]: cluster 2023-10-09T04:43:34.570448+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v927: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 31406/249201 objects degraded (12.603%) 2023-10-09T04:43:36.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:36.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:36.421 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:36.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:36.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:36.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:36.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 87517 87501 95.9328 65.5 0.0109444 0.0104184 2023-10-09T04:43:36.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:36.881 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:36.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:36.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:37.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:37.144 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:37.168 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:37.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:37.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:37.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:37.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:37.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:37.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:37.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 88585 88569 95.4296 66.75 0.0298726 0.0104729 2023-10-09T04:43:37.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:37.856 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:37.960 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:37.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:38.022 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:38 smithi153 bash[59565]: cluster 2023-10-09T04:43:36.571124+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v928: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 32050/255171 objects degraded (12.560%) 2023-10-09T04:43:38.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:38.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:38.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:38.220 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:38.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:38 smithi082 bash[77063]: cluster 2023-10-09T04:43:36.571124+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v928: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 32050/255171 objects degraded (12.560%) 2023-10-09T04:43:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:38 smithi082 bash[79733]: cluster 2023-10-09T04:43:36.571124+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v928: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 32050/255171 objects degraded (12.560%) 2023-10-09T04:43:38.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:38.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:38.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:38.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:38.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:38.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:38.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:38.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 89413 89397 94.6892 51.75 0.018563 0.0105548 2023-10-09T04:43:38.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:38.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:38.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:39.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:39.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:39.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:39.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:39.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:39.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:39.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:39.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:39.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:39.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:39.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:43:39.717011+0000 min lat: 0.00125881 max lat: 0.429926 avg lat: 0.0106362 2023-10-09T04:43:39.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:43:39.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 90234 90218 93.9662 51.3125 0.0143012 0.0106362 2023-10-09T04:43:39.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:39.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:39.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:39.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:39.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:40.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:40.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:40 smithi153 bash[59565]: cluster 2023-10-09T04:43:38.571949+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v929: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 32382/259998 objects degraded (12.455%) 2023-10-09T04:43:40.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:40 smithi153 bash[59565]: cluster 2023-10-09T04:43:39.914247+0000 mon.a (mon.0) 1311 : cluster 3 Health check update: Degraded data redundancy: 32382/259998 objects degraded (12.455%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:40.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:40.206 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:40 smithi082 bash[77063]: cluster 2023-10-09T04:43:38.571949+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v929: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 32382/259998 objects degraded (12.455%) 2023-10-09T04:43:40.206 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:40 smithi082 bash[77063]: cluster 2023-10-09T04:43:39.914247+0000 mon.a (mon.0) 1311 : cluster 3 Health check update: Degraded data redundancy: 32382/259998 objects degraded (12.455%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:40.207 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:40 smithi082 bash[79733]: cluster 2023-10-09T04:43:38.571949+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v929: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 32382/259998 objects degraded (12.455%) 2023-10-09T04:43:40.207 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:40 smithi082 bash[79733]: cluster 2023-10-09T04:43:39.914247+0000 mon.a (mon.0) 1311 : cluster 3 Health check update: Degraded data redundancy: 32382/259998 objects degraded (12.455%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:40.207 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:39] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T04:43:40.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:40.328 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:40.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:40.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:40.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:40.602 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:40.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:40.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 91129 91113 93.3427 55.9375 0.0222123 0.0107061 2023-10-09T04:43:40.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:40.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:40.881 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:40.909 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:40.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:41.444 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:43:41.448 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:41.477 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:43:41.514 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:43:41.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:41.555 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:43:41.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:41.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:41.605 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:43:41.635 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.674 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.675 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.675 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:43:41.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:40.501662+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:40.509486+0000 mon.a (mon.0) 1313 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:40.527289+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:40.536769+0000 mon.a (mon.0) 1315 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: cluster 2023-10-09T04:43:40.572774+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v930: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:41.176864+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:41.177743+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:41.183545+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:41 smithi153 bash[59565]: audit 2023-10-09T04:43:41.367572+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.82:0/1599508627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.687 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:41.717 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 92017 92001 92.7323 55.5 0.00604087 0.0107783 2023-10-09T04:43:41.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:41.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:40.501662+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:40.509486+0000 mon.a (mon.0) 1313 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:40.527289+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.810 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:40.536769+0000 mon.a (mon.0) 1315 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: cluster 2023-10-09T04:43:40.572774+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v930: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:41.176864+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:41.177743+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:41.183545+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[77063]: audit 2023-10-09T04:43:41.367572+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.82:0/1599508627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:40.501662+0000 mon.a (mon.0) 1312 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:40.509486+0000 mon.a (mon.0) 1313 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:40.527289+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.811 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:40.536769+0000 mon.a (mon.0) 1315 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: cluster 2023-10-09T04:43:40.572774+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v930: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:41.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:41.176864+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:41.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:41.177743+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:41.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:41.183545+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:43:41.812 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:41 smithi082 bash[79733]: audit 2023-10-09T04:43:41.367572+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.82:0/1599508627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:41.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:42.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:42.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:42.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:42.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:42.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:42.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:42.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:42.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:42.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:42.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:42.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:42.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:42.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 93060 93044 92.2949 65.1875 0.020553 0.0108295 2023-10-09T04:43:42.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:42.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:42.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:42.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:42.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:43.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:43.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:43.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:43.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:43.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:43.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:43.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:43.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:43.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:43.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:43.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:43.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 94042 94026 91.8117 61.375 0.0212437 0.0108862 2023-10-09T04:43:43.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:43.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:43 smithi082 bash[77063]: cluster 2023-10-09T04:43:42.573560+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v931: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:43.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:43 smithi082 bash[79733]: cluster 2023-10-09T04:43:42.573560+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v931: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:43.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:43.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:43.878 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:43.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:43.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:43 smithi153 bash[59565]: cluster 2023-10-09T04:43:42.573560+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v931: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 33874/269925 objects degraded (12.549%) 2023-10-09T04:43:43.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:44.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:44.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:44.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:44.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:44.264 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:44.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:44.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:44.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:44.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:44.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:44.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:44.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:44.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:44.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 94923 94907 91.2463 55.0625 0.00794623 0.0109525 2023-10-09T04:43:44.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:44.827 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:44 smithi082 bash[79733]: audit 2023-10-09T04:43:44.283682+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:44.827 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:44 smithi082 bash[77063]: audit 2023-10-09T04:43:44.283682+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:44.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:44.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:44 smithi153 bash[59565]: audit 2023-10-09T04:43:44.283682+0000 mon.a (mon.0) 1319 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:44.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:44.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:44.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:44.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:45.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:45.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:45.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:45.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:45.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:45.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:45.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:45.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:45.649 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:45.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 95879 95863 90.769 59.75 0.0166635 0.0110117 2023-10-09T04:43:45.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:45.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.923 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:45 smithi082 bash[79733]: cluster 2023-10-09T04:43:44.574388+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v932: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 35033/278634 objects degraded (12.573%) 2023-10-09T04:43:45.923 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:45 smithi082 bash[79733]: cluster 2023-10-09T04:43:44.915182+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 35033/278634 objects degraded (12.573%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:45.923 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:45 smithi082 bash[77063]: cluster 2023-10-09T04:43:44.574388+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v932: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 35033/278634 objects degraded (12.573%) 2023-10-09T04:43:45.923 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:45 smithi082 bash[77063]: cluster 2023-10-09T04:43:44.915182+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 35033/278634 objects degraded (12.573%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:45 smithi153 bash[59565]: cluster 2023-10-09T04:43:44.574388+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v932: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 35033/278634 objects degraded (12.573%) 2023-10-09T04:43:45.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:45 smithi153 bash[59565]: cluster 2023-10-09T04:43:44.915182+0000 mon.a (mon.0) 1320 : cluster 3 Health check update: Degraded data redundancy: 35033/278634 objects degraded (12.573%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:45.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:45.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:43:45.985 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:43:46.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:46.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:46.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:46.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:46.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:46.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:46.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 96883 96867 90.3507 62.75 0.0144125 0.0110624 2023-10-09T04:43:46.748 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:46.852 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:46.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:46.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:46.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:47.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:47.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:47.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:47.238 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.263 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:47.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:47.503 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:47.528 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:47.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:47.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:47.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 97817 97801 89.8804 58.375 0.0175341 0.0111209 2023-10-09T04:43:47.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:47.774 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:47 smithi082 bash[79733]: cluster 2023-10-09T04:43:46.575204+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v933: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s; 35624/283917 objects degraded (12.547%) 2023-10-09T04:43:47.775 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:47 smithi082 bash[77063]: cluster 2023-10-09T04:43:46.575204+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v933: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s; 35624/283917 objects degraded (12.547%) 2023-10-09T04:43:47.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:47.900 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:47.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:47 smithi153 bash[59565]: cluster 2023-10-09T04:43:46.575204+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v933: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s; 35624/283917 objects degraded (12.547%) 2023-10-09T04:43:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:47.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:47.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:43:48.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:48.089 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:48.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:48.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:48.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:48.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:48.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:48.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:48.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:48.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:48.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:48.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 16 98775 98759 89.4455 59.875 0.00775606 0.0111731 2023-10-09T04:43:48.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:48.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:48.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:48.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:48.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:49.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:49.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:49.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:49.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:49.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:49.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:49.600 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:49.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:49.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:49.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 99673 99657 88.9694 56.125 0.0143732 0.0112342 2023-10-09T04:43:49.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:49.747 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:49 smithi082 bash[77063]: cluster 2023-10-09T04:43:48.575936+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v934: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 35624/285684 objects degraded (12.470%) 2023-10-09T04:43:49.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:49.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:49.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:49 smithi153 bash[59565]: cluster 2023-10-09T04:43:48.575936+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v934: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 35624/285684 objects degraded (12.470%) 2023-10-09T04:43:49.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:49.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:50.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:49 smithi082 bash[79733]: cluster 2023-10-09T04:43:48.575936+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v934: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 35624/285684 objects degraded (12.470%) 2023-10-09T04:43:50.000 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:49] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-10-09T04:43:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:50.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:50.133 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:50.236 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:50.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:50.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:50.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:50.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:50.547 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:50.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:50.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:50.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:50.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 15 100589 100574 88.5235 57.3125 0.0196771 0.0112911 2023-10-09T04:43:50.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:50.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:50.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:50.922 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:50 smithi082 bash[77063]: cluster 2023-10-09T04:43:49.918291+0000 mon.a (mon.0) 1321 : cluster 3 Health check update: Degraded data redundancy: 35624/285684 objects degraded (12.470%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:50.922 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:50 smithi082 bash[79733]: cluster 2023-10-09T04:43:49.918291+0000 mon.a (mon.0) 1321 : cluster 3 Health check update: Degraded data redundancy: 35624/285684 objects degraded (12.470%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:50.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:50 smithi153 bash[59565]: cluster 2023-10-09T04:43:49.918291+0000 mon.a (mon.0) 1321 : cluster 3 Health check update: Degraded data redundancy: 35624/285684 objects degraded (12.470%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:50.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:50.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:50.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:51.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:51.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:51.187 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:51.213 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:51.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:51.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:51.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:51.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:51.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:51.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:51.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 16 101548 101532 88.1255 59.875 0.00990838 0.0113415 2023-10-09T04:43:51.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:51.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:51.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:51.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:51.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:51.964 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:43:51.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:51 smithi153 bash[59565]: cluster 2023-10-09T04:43:50.576810+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v935: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:52.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:51 smithi153 bash[59565]: audit 2023-10-09T04:43:51.417585+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.82:0/994693094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.008 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:51 smithi153 bash[59565]: audit 2023-10-09T04:43:51.419134+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.023 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:52.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:52.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[77063]: cluster 2023-10-09T04:43:50.576810+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v935: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:52.141 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[77063]: audit 2023-10-09T04:43:51.417585+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.82:0/994693094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.141 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[77063]: audit 2023-10-09T04:43:51.419134+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.141 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[79733]: cluster 2023-10-09T04:43:50.576810+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v935: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:52.141 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[79733]: audit 2023-10-09T04:43:51.417585+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.82:0/994693094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.141 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:51 smithi082 bash[79733]: audit 2023-10-09T04:43:51.419134+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:52.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:52.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:52.233 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:43:52.234 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:43:52.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:52.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:52.411 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:52.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:52.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.567 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:52.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:52.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:52.718 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 102445 102429 87.6862 56.0625 0.0135124 0.0113988 2023-10-09T04:43:52.802 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:52.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:52.952 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 0.982833 (fb9a) 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.924616+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.941595+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.944565+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: audit 2023-10-09T04:43:51.946550+0000 mon.a (mon.0) 1324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.946873+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e226: 8 total, 7 up, 8 in 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.948382+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.958 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: cluster 2023-10-09T04:43:51.950603+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: audit 2023-10-09T04:43:52.596398+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.82:0/3375723600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[77063]: audit 2023-10-09T04:43:52.597118+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.924616+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.941595+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.944565+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: audit 2023-10-09T04:43:51.946550+0000 mon.a (mon.0) 1324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.946873+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e226: 8 total, 7 up, 8 in 2023-10-09T04:43:52.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.948382+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.960 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: cluster 2023-10-09T04:43:51.950603+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:52.960 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: audit 2023-10-09T04:43:52.596398+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.82:0/3375723600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:52.960 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:52 smithi082 bash[79733]: audit 2023-10-09T04:43:52.597118+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:52.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:52.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:53.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:53.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.924616+0000 mon.a (mon.0) 1323 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.941595+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.944565+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: audit 2023-10-09T04:43:51.946550+0000 mon.a (mon.0) 1324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.946873+0000 mon.a (mon.0) 1325 : cluster 0 osdmap e226: 8 total, 7 up, 8 in 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.948382+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: cluster 2023-10-09T04:43:51.950603+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: audit 2023-10-09T04:43:52.596398+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.82:0/3375723600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:52 smithi153 bash[59565]: audit 2023-10-09T04:43:52.597118+0000 mon.a (mon.0) 1326 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]: dispatch 2023-10-09T04:43:53.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:53.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:53.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:53.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:53.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:53.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:53.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:53.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:53.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:53.719 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 103379 103363 87.2901 58.375 0.0108965 0.0114503 2023-10-09T04:43:53.748 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:53.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:53.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:53.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.906 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:53.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:54.009 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:54.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:54.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.577596+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v937: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.944359+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.945898+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: audit 2023-10-09T04:43:52.946110+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]': finished 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.946322+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e227: 8 total, 7 up, 8 in 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.956347+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.956636+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.956902+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[77063]: cluster 2023-10-09T04:43:52.958985+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.577596+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v937: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.944359+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.945898+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: audit 2023-10-09T04:43:52.946110+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]': finished 2023-10-09T04:43:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.946322+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e227: 8 total, 7 up, 8 in 2023-10-09T04:43:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.956347+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.956636+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.956902+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:53 smithi082 bash[79733]: cluster 2023-10-09T04:43:52.958985+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.577596+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v937: 89 pgs: 9 active+undersized+degraded, 18 active+undersized, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 37395/297984 objects degraded (12.549%) 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.944359+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.945898+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: audit 2023-10-09T04:43:52.946110+0000 mon.a (mon.0) 1327 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.982832888051702}]': finished 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.946322+0000 mon.a (mon.0) 1328 : cluster 0 osdmap e227: 8 total, 7 up, 8 in 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.956347+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.956636+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.956902+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:53 smithi153 bash[59565]: cluster 2023-10-09T04:43:52.958985+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T04:43:54.302 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:54.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:54.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:54.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:54.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:54.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.719 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 104263 104247 86.8628 55.25 0.0290957 0.0115072 2023-10-09T04:43:54.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:54.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:54.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:54.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:54.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:55.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:55.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:55.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:55.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.954461+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.957672+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.959767+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e228: 8 total, 7 up, 8 in 2023-10-09T04:43:55.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.963538+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.199 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.963662+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.965382+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:53.965645+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.919537+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 37395/297984 objects degraded (12.549%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:55.201 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:43:55.201 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.954461+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.201 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.957672+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.959767+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e228: 8 total, 7 up, 8 in 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.963538+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.963662+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.965382+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:53.965645+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:54 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.919537+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 37395/297984 objects degraded (12.549%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:55.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:55.250 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.954461+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.957672+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.959767+0000 mon.a (mon.0) 1329 : cluster 0 osdmap e228: 8 total, 7 up, 8 in 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.963538+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.963662+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.965382+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:53.965645+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T04:43:55.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:54 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.919537+0000 mon.a (mon.0) 1330 : cluster 3 Health check update: Degraded data redundancy: 37395/297984 objects degraded (12.549%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:55.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:55.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:55.415 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:55.520 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:55.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:55.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:55.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:55.697 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:55.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 105173 105157 86.4652 56.875 0.0169135 0.0115599 2023-10-09T04:43:55.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:55.820 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:55.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:55.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:55.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:55.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.578334+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v940: 89 pgs: 1 peering, 9 active+undersized+degraded, 18 active+undersized, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 38546/306738 objects degraded (12.566%) 2023-10-09T04:43:55.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.964985+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:55.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.967864+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:55.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.968716+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e229: 8 total, 7 up, 8 in 2023-10-09T04:43:55.970 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[79733]: cluster 2023-10-09T04:43:54.971512+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:55.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.578334+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v940: 89 pgs: 1 peering, 9 active+undersized+degraded, 18 active+undersized, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 38546/306738 objects degraded (12.566%) 2023-10-09T04:43:55.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.964985+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:55.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.967864+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:55.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.968716+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e229: 8 total, 7 up, 8 in 2023-10-09T04:43:55.971 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:55 smithi082 bash[77063]: cluster 2023-10-09T04:43:54.971512+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:56.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:56.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:56.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:56.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:56.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:55 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.578334+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v940: 89 pgs: 1 peering, 9 active+undersized+degraded, 18 active+undersized, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 38546/306738 objects degraded (12.566%) 2023-10-09T04:43:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:55 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.964985+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:55 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.967864+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:55 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.968716+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e229: 8 total, 7 up, 8 in 2023-10-09T04:43:56.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:55 smithi153 bash[59565]: cluster 2023-10-09T04:43:54.971512+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T04:43:56.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:56.491 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:43:56.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:56.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:56.634 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:56.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 16 106106 106090 86.0996 58.3125 0.0317727 0.0116089 2023-10-09T04:43:56.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:56.873 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:56.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:57.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.032 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:57.137 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:57.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:57.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:57.435 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:57.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:57.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:57.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:57.716 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:57.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 107041 107025 85.7449 58.4375 0.0121919 0.0116572 2023-10-09T04:43:57.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:57.850 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:57.957 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:57.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:57.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:58.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:58.095 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:57 smithi082 bash[79733]: cluster 2023-10-09T04:43:56.579317+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v942: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 39121/311910 objects degraded (12.542%); 0 B/s, 0 objects/s recovering 2023-10-09T04:43:58.096 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:57 smithi082 bash[77063]: cluster 2023-10-09T04:43:56.579317+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v942: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 39121/311910 objects degraded (12.542%); 0 B/s, 0 objects/s recovering 2023-10-09T04:43:58.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:58.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:58.225 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:57 smithi153 bash[59565]: cluster 2023-10-09T04:43:56.579317+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v942: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 39121/311910 objects degraded (12.542%); 0 B/s, 0 objects/s recovering 2023-10-09T04:43:58.243 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:58.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:58.385 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:58.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:58.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.664 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:58.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 107955 107939 85.3825 57.125 0.0123366 0.0117062 2023-10-09T04:43:58.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:58.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:58.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.931 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:58.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:59.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:59.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:59.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:59.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:59.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:59.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:59.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:59.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:59.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:59.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:59.702 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:59.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:43:59.721445+0000 min lat: 0.00125881 max lat: 0.429926 avg lat: 0.0117551 2023-10-09T04:43:59.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:43:59.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 108869 108853 85.0293 57.125 0.0217502 0.0117551 2023-10-09T04:43:59.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:59.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:43:59] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T04:43:59.763 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:59.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:59.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:43:59.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:59.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.003 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:00.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:00.226 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 7, 6, 5] 2023-10-09T04:44:00.226 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:44:00.227 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:44:00.227 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.926338 2023-10-09T04:44:00.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.9263379889292654 2023-10-09T04:44:00.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.247 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:59 smithi153 bash[59565]: cluster 2023-10-09T04:43:58.580192+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v943: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 39121/313569 objects degraded (12.476%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:00.248 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:59 smithi153 bash[59565]: audit 2023-10-09T04:43:59.283508+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:00.248 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:43:59 smithi153 bash[59565]: cluster 2023-10-09T04:43:59.920763+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 39121/313569 objects degraded (12.476%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:00.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[77063]: cluster 2023-10-09T04:43:58.580192+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v943: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 39121/313569 objects degraded (12.476%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[77063]: audit 2023-10-09T04:43:59.283508+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[77063]: cluster 2023-10-09T04:43:59.920763+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 39121/313569 objects degraded (12.476%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[79733]: cluster 2023-10-09T04:43:58.580192+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v943: 89 pgs: 2 active+clean+remapped, 1 peering, 9 active+undersized+degraded, 14 active+undersized, 63 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 39121/313569 objects degraded (12.476%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[79733]: audit 2023-10-09T04:43:59.283508+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:00.310 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:43:59 smithi082 bash[79733]: cluster 2023-10-09T04:43:59.920763+0000 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 39121/313569 objects degraded (12.476%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:00.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:00.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:00.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:00.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 109712 109696 84.6299 52.6875 0.0132209 0.0118108 2023-10-09T04:44:00.779 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:00.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:00.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:00.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:00.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:00.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:01.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:01.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:01.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:01.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:01.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:01.383 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:01.487 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:01.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:01.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:01.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:01.708 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:44:01.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 110601 110585 84.2754 55.5625 0.00959687 0.0118608 2023-10-09T04:44:01.749 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:44:01.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:01.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:01.806 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:44:01.842 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:44:01.878 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:44:01.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:01.911 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:44:01.931 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:01.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:01.934 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:44:01.934 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.934 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:44:01.934 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.935 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.935 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.935 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:44:01.935 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:44:02.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:02.042 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:02 smithi082 bash[77063]: cluster 2023-10-09T04:44:00.581162+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:02.042 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:02 smithi082 bash[77063]: audit 2023-10-09T04:44:01.671006+0000 mon.a (mon.0) 1334 : audit 0 from='client.? 172.21.15.82:0/553806428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:02.042 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:02 smithi082 bash[79733]: cluster 2023-10-09T04:44:00.581162+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:02.042 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:02 smithi082 bash[79733]: audit 2023-10-09T04:44:01.671006+0000 mon.a (mon.0) 1334 : audit 0 from='client.? 172.21.15.82:0/553806428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:02.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:02.165 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:02.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:01 smithi153 bash[59565]: cluster 2023-10-09T04:44:00.581162+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v944: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:02.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:01 smithi153 bash[59565]: audit 2023-10-09T04:44:01.671006+0000 mon.a (mon.0) 1334 : audit 0 from='client.? 172.21.15.82:0/553806428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:02.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:02.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:02.336 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:02.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:02.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:02.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:02.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:02.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:02.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:02.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 16 111447 111431 83.897 52.875 0.0145476 0.0119134 2023-10-09T04:44:02.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:02.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:02.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:02.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:02.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:02.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:03.026 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:03.129 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:03.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:03.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:03.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:03.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:03.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:03.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:03.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:03.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:03.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:03.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 112241 112225 83.489 49.625 0.00433912 0.0119571 2023-10-09T04:44:03.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:03.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:03.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:03.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:03.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:04.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:04.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:04.215 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.223 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:03 smithi082 bash[77063]: cluster 2023-10-09T04:44:02.581896+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v945: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:04.223 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:03 smithi082 bash[79733]: cluster 2023-10-09T04:44:02.581896+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v945: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:04.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:04.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:04.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:03 smithi153 bash[59565]: cluster 2023-10-09T04:44:02.581896+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v945: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 40883/325650 objects degraded (12.554%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:04.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:04.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:04.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:04.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:04.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 112929 112913 83.0126 43 0.00887124 0.0120294 2023-10-09T04:44:04.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:04.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:04.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:04.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:04.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:05.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.037 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:04 smithi153 bash[59565]: cluster 2023-10-09T04:44:04.921933+0000 mon.a (mon.0) 1335 : cluster 3 Health check update: Degraded data redundancy: 40883/325650 objects degraded (12.554%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:05.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:04 smithi082 bash[77063]: cluster 2023-10-09T04:44:04.921933+0000 mon.a (mon.0) 1335 : cluster 3 Health check update: Degraded data redundancy: 40883/325650 objects degraded (12.554%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:05.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:04 smithi082 bash[79733]: cluster 2023-10-09T04:44:04.921933+0000 mon.a (mon.0) 1335 : cluster 3 Health check update: Degraded data redundancy: 40883/325650 objects degraded (12.554%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:05.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:05.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:05.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:05.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:05.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:05.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:05.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:05.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 113685 113669 82.5967 47.25 0.0072601 0.0120947 2023-10-09T04:44:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:05.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:05.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:05.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:05.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:05.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:05.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:44:06.058 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:44:06.094 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:06.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:06.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:06.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:05 smithi082 bash[77063]: cluster 2023-10-09T04:44:04.582601+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v946: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 41903/333651 objects degraded (12.559%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:06.306 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:05 smithi082 bash[79733]: cluster 2023-10-09T04:44:04.582601+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v946: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 41903/333651 objects degraded (12.559%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:06.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:06.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:06 smithi153 bash[59565]: cluster 2023-10-09T04:44:04.582601+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v946: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 41903/333651 objects degraded (12.559%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:06.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:06.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:06.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:06.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 114448 114432 82.1954 47.6875 0.0197144 0.0121607 2023-10-09T04:44:06.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:06.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:06.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:06.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:06.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:07.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:07.065 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:07.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:07.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:07.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:07.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.448 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:07.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:07.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:07.709 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:07.722 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 115336 115320 81.892 55.5 0.00990049 0.0122054 2023-10-09T04:44:07.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:07.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:07.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:08.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:44:08.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:08.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:08 smithi153 bash[59565]: cluster 2023-10-09T04:44:06.583272+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 42405/338379 objects degraded (12.532%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:08.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:08.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:08.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:08 smithi082 bash[77063]: cluster 2023-10-09T04:44:06.583272+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 42405/338379 objects degraded (12.532%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:08.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:08 smithi082 bash[79733]: cluster 2023-10-09T04:44:06.583272+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v947: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 42405/338379 objects degraded (12.532%); 0 B/s, 0 objects/s recovering 2023-10-09T04:44:08.266 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:08.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:08.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:08.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:08.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:08.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:08.650 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:08.675 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:08.723 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 16 116186 116170 81.5687 53.125 0.0055469 0.0122541 2023-10-09T04:44:08.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:08.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:08.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:08.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:08.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:08.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:09.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:09.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:09.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:09.199 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:09.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:09.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:09.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:09.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:09.723 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 90 9 117011 117002 81.2402 52 0.0213422 0.0123031 2023-10-09T04:44:09.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:09.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:09.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:09.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:09] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T04:44:09.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:10.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:10.123 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:10.136 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:10 smithi153 bash[59565]: cluster 2023-10-09T04:44:08.583986+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v948: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 42405/339813 objects degraded (12.479%) 2023-10-09T04:44:10.137 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:10 smithi153 bash[59565]: cluster 2023-10-09T04:44:09.923210+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 42405/339813 objects degraded (12.479%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:10.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:10.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:10.210 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144248 7.19 7:9df30e0f:::benchmark_data_smithi082_155217_object41588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:10.764147+0000", 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.44219156299999, 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6324767859999998, 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.211 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144248 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764147+0000", 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764147+0000", 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764146+0000", 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764149+0000", 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8009999999999999e-06 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764155+0000", 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1070000000000001e-06 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764160+0000", 2023-10-09T04:44:10.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1499999999999998e-06 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.764206+0000", 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5893999999999998e-05 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for degraded object", 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.765115+0000", 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00090880500000000005 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.980337+0000", 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.215222583 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.980439+0000", 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010207199999999999 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:44:10.216 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.980893+0000", 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00045411300000000002 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.011912+0000", 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031018295000000001 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.011924+0000", 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2289999999999999e-05 2023-10-09T04:44:10.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.802075+0000", 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.79015054200000001 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.802086+0000", 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1793e-05 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.282722+0000", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4806352309999999 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for missing object", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.282868+0000", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014646400000000001 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.396503+0000", 2023-10-09T04:44:10.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.113635287 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.396623+0000", 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011995 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.220 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144251 7.f 7:f744365f:::benchmark_data_smithi082_155217_object41591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:10.768671+0000", 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.43766740199999, 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.357225444, 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144251 2023-10-09T04:44:10.221 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768671+0000", 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768671+0000", 2023-10-09T04:44:10.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768670+0000", 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768673+0000", 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9170000000000002e-06 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768682+0000", 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7309999999999999e-06 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.768695+0000", 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3047999999999999e-05 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.982781+0000", 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21408649299999999 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.982917+0000", 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000135191 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.983037+0000", 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000120321 2023-10-09T04:44:10.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.009928+0000", 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.026890898999999999 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.010031+0000", 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010303500000000001 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.226 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.800780+0000", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.79074889100000001 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.800797+0000", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7411999999999999e-05 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.094453+0000", 2023-10-09T04:44:10.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2936561529999999 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for missing object", 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.094661+0000", 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000207603 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.125808+0000", 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031146930999999999 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.125896+0000", 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8368000000000005e-05 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144257 7.6 7:64892055:::benchmark_data_smithi082_155217_object41597:head [delete] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:10.976980+0000", 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.22935768599999, 2023-10-09T04:44:10.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2478133090000001, 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144257 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.976980+0000", 2023-10-09T04:44:10.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.976980+0000", 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.976979+0000", 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.976987+0000", 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9869999999999999e-06 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977001+0000", 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4164e-05 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977009+0000", 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2369999999999992e-06 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.985267+0000", 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0082582630000000001 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.233 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.985411+0000", 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014305700000000001 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.985948+0000", 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00053738899999999999 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.013673+0000", 2023-10-09T04:44:10.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027725052 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.013685+0000", 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2332e-05 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.801078+0000", 2023-10-09T04:44:10.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.78739297600000002 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.801091+0000", 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3023000000000001e-05 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.202105+0000", 2023-10-09T04:44:10.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.401013216 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for missing object", 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.202260+0000", 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000155787 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.224699+0000", 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.022438356 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.224794+0000", 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5041e-05 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.238 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144259 7.f 7:f7e46343:::benchmark_data_smithi082_155217_object41599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e188)", 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:10.977017+0000", 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.22932118099999, 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1493724790000002, 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.239 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144259 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977017+0000", 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977017+0000", 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977016+0000", 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977020+0000", 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.2289999999999999e-06 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977026+0000", 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8789999999999997e-06 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.977032+0000", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5010000000000004e-06 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.990532+0000", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013500010999999999 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.990690+0000", 2023-10-09T04:44:10.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000158326 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:10.995686+0000", 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049953230000000003 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.012814+0000", 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017128833999999999 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.012827+0000", 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2218000000000001e-05 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.800833+0000", 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.78800610699999996 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.245 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.800845+0000", 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2638e-05 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.094958+0000", 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2941127940000001 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for missing object", 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.095120+0000", 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016223600000000001 2023-10-09T04:44:10.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.126325+0000", 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031204619999999999 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.126389+0000", 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4302999999999997e-05 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.247 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144272 7.6 7:61f3e6de:::benchmark_data_smithi082_155217_object41612:head [delete] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:11.019847+0000", 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.18649151299999, 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.219372855, 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.248 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144272 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019847+0000", 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019847+0000", 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019845+0000", 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019852+0000", 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4139999999999997e-06 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019867+0000", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4745e-05 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019876+0000", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6729999999999998e-06 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019938+0000", 2023-10-09T04:44:10.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1826000000000001e-05 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.019951+0000", 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3011e-05 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.801159+0000", 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.78120795300000001 2023-10-09T04:44:10.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.801167+0000", 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1119999999999996e-06 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.225479+0000", 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.424312797 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.253 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.225761+0000", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00028200700000000003 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.226226+0000", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00046438600000000001 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.235601+0000", 2023-10-09T04:44:10.254 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0093751459999999991 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.235785+0000", 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018432999999999999 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.237536+0000", 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0017508000000000001 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.237584+0000", 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8065000000000001e-05 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.239219+0000", 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0016352230000000001 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:144281 7.19 7:9c3b904e:::benchmark_data_smithi082_155217_object41621:head [delete] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:40:11.033129+0000", 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 239.17320945899999, 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3831175070000001, 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.257 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 144281 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033129+0000", 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033129+0000", 2023-10-09T04:44:10.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033127+0000", 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033135+0000", 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7419999999999997e-06 2023-10-09T04:44:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033148+0000", 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2847000000000001e-05 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033155+0000", 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1570000000000003e-06 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033220+0000", 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5331000000000002e-05 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.033234+0000", 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3807e-05 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.261 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.802113+0000", 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.76887950000000005 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:11.802122+0000", 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4139999999999994e-06 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.396776+0000", 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5946543520000001 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.262 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.396909+0000", 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013290399999999999 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.397369+0000", 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00046013599999999999 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.263 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.399336+0000", 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019666549999999999 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.414172+0000", 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.014835951 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.265 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.414198+0000", 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6103999999999999e-05 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.414237+0000", 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8943999999999998e-05 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.266 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:40:13.416246+0000", 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0020092310000000002 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201331 7.6 7:6141402a:::benchmark_data_smithi082_155217_object98671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.703651+0000", 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.502687006, 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0603340889999999, 2023-10-09T04:44:10.267 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201331 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703651+0000", 2023-10-09T04:44:10.268 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703651+0000", 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703650+0000", 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:10.269 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703655+0000", 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.673e-06 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703664+0000", 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1949999999999992e-06 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.703669+0000", 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3029999999999999e-06 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705636+0000", 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0019666869999999999 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.271 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705775+0000", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013892399999999999 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.706403+0000", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00062812399999999998 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.723881+0000", 2023-10-09T04:44:10.272 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017478074999999999 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.725436+0000", 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0015552299999999999 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.750856+0000", 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.025419674 2023-10-09T04:44:10.273 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.750868+0000", 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2215000000000001e-05 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.763900+0000", 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0130315379999999 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.274 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.763985+0000", 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5371999999999995e-05 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201333 7.6 7:67eadb8e:::benchmark_data_smithi082_155217_object98673:head [delete] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.705269+0000", 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.50106860299999, 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0593016930000001, 2023-10-09T04:44:10.275 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201333 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705269+0000", 2023-10-09T04:44:10.276 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705269+0000", 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705269+0000", 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.277 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705274+0000", 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.831e-06 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705284+0000", 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0348e-05 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.278 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.705292+0000", 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8169999999999999e-06 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.716447+0000", 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011155351000000001 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.716585+0000", 2023-10-09T04:44:10.279 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000138348 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.718273+0000", 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001687912 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.725473+0000", 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0071997509999999999 2023-10-09T04:44:10.280 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.750932+0000", 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.025458668 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.750940+0000", 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6789999999999994e-06 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.281 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.764512+0000", 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.013571802 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.764571+0000", 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8897000000000002e-05 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.282 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201367 7.6 7:668e72ac:::benchmark_data_smithi082_155217_object98707:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.760680+0000", 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.445657625, 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0217613759999999, 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201367 2023-10-09T04:44:10.283 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760680+0000", 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760680+0000", 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.284 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760679+0000", 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760687+0000", 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.622e-06 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.285 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760698+0000", 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1127e-05 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760705+0000", 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.985e-06 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.286 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760958+0000", 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00025369599999999998 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.760965+0000", 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.63e-06 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.287 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.764591+0000", 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.003626012 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.764684+0000", 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2825000000000007e-05 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.765271+0000", 2023-10-09T04:44:10.288 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00058741999999999996 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.775809+0000", 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010537695 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782262+0000", 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064534570000000001 2023-10-09T04:44:10.289 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782294+0000", 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1179000000000001e-05 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782395+0000", 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000101537 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782442+0000", 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6631000000000001e-05 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201370 7.6 7:64813969:::benchmark_data_smithi082_155217_object98710:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.761344+0000", 2023-10-09T04:44:10.291 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.44499400000001, 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.036947579, 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201370 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.292 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761344+0000", 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761344+0000", 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761343+0000", 2023-10-09T04:44:10.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761349+0000", 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.088e-06 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761361+0000", 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.167e-05 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761368+0000", 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5460000000000003e-06 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761413+0000", 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4552000000000003e-05 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.761422+0000", 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1789999999999997e-06 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.768192+0000", 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.006770239 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.768304+0000", 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000111425 2023-10-09T04:44:10.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.768746+0000", 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00044206099999999999 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782478+0000", 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013731672 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.785909+0000", 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034319519999999998 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798228+0000", 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012318068 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798263+0000", 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5160000000000002e-05 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798292+0000", 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8966999999999999e-05 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201444 7.6 7:66eb3da5:::benchmark_data_smithi082_155217_object98784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.863813+0000", 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.34252462200001, 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.934547354, 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201444 2023-10-09T04:44:10.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863813+0000", 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863813+0000", 2023-10-09T04:44:10.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863812+0000", 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863819+0000", 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4939999999999997e-06 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863833+0000", 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3658e-05 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863844+0000", 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0519e-05 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863897+0000", 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3136000000000001e-05 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.863910+0000", 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.344e-05 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.772330+0000", 2023-10-09T04:44:10.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.90841976999999996 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.772422+0000", 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1774000000000006e-05 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.772931+0000", 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00050889700000000002 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782506+0000", 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0095753910000000008 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.785972+0000", 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034658269999999999 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798316+0000", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012344232 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798339+0000", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3071999999999999e-05 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.798361+0000", 2023-10-09T04:44:10.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1630000000000001e-05 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201491 7.6 7:64c56703:::benchmark_data_smithi082_155217_object98831:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.924763+0000", 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.28157469300001, 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.87906602700000003, 2023-10-09T04:44:10.308 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201491 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924763+0000", 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924763+0000", 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924763+0000", 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924767+0000", 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9260000000000002e-06 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924782+0000", 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5196e-05 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.924797+0000", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5520999999999999e-05 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.925432+0000", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00063427700000000004 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.925442+0000", 2023-10-09T04:44:10.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0756e-05 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.775835+0000", 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.85039298100000005 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.775937+0000", 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000102039 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.776427+0000", 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00048948100000000005 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782535+0000", 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0061077179999999998 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.785978+0000", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034431610000000001 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.803507+0000", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017528823999999998 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.803665+0000", 2023-10-09T04:44:10.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015885199999999999 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.803829+0000", 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016400299999999999 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201495 7.6 7:64e35616:::benchmark_data_smithi082_155217_object98835:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.316 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.930447+0000", 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.27589144999999, 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.87404892000000001, 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201495 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930447+0000", 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930447+0000", 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930446+0000", 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930451+0000", 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1080000000000001e-06 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930460+0000", 2023-10-09T04:44:10.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7020000000000007e-06 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930467+0000", 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2579999999999998e-06 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930515+0000", 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8266999999999997e-05 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.930526+0000", 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0397000000000001e-05 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.779164+0000", 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.84863825299999995 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.779269+0000", 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000104681 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.779692+0000", 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00042370100000000001 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.785963+0000", 2023-10-09T04:44:10.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0062711279999999999 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.785984+0000", 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0295000000000001e-05 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804191+0000", 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.018206875000000001 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804340+0000", 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000149419 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804496+0000", 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015563999999999999 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201501 7.6 7:66d6c070:::benchmark_data_smithi082_155217_object98841:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:35.938379+0000", 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.26795869200001, 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.86632197899999996, 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.325 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201501 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938379+0000", 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938379+0000", 2023-10-09T04:44:10.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938378+0000", 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938383+0000", 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3700000000000003e-06 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938392+0000", 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1030000000000008e-06 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.938398+0000", 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.344e-06 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.940997+0000", 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0025992139999999999 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:35.941008+0000", 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1549000000000001e-05 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782555+0000", 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.84154681799999997 2023-10-09T04:44:10.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.782672+0000", 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011654600000000001 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.783151+0000", 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000479703 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.800160+0000", 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017008921999999999 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.800217+0000", 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6252000000000003e-05 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804619+0000", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.004402156 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804662+0000", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3334999999999998e-05 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804701+0000", 2023-10-09T04:44:10.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9233000000000002e-05 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201563 7.6 7:618471c7:::benchmark_data_smithi082_155217_object98903:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:36.030911+0000", 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.17542695500001, 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.78078042800000003, 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.333 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201563 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030911+0000", 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030911+0000", 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030910+0000", 2023-10-09T04:44:10.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030916+0000", 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3409999999999999e-06 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030928+0000", 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2061e-05 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.030948+0000", 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9907000000000001e-05 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.031007+0000", 2023-10-09T04:44:10.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8386e-05 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.031021+0000", 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4389999999999999e-05 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.786014+0000", 2023-10-09T04:44:10.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.75499338999999999 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.786129+0000", 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011445100000000001 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.786582+0000", 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00045319400000000003 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804012+0000", 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017430438999999999 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804757+0000", 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00074424400000000003 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811597+0000", 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068402580000000001 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811653+0000", 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5692999999999997e-05 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811692+0000", 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8883e-05 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.342 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35259.0:201596 7.6 7:639b03c6:::benchmark_data_smithi082_155217_object98936:head [delete] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:41:36.084406+0000", 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 154.121931945, 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.72737576400000004, 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.343 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35259", 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 201596 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084406+0000", 2023-10-09T04:44:10.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084406+0000", 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084405+0000", 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084410+0000", 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8940000000000001e-06 2023-10-09T04:44:10.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084420+0000", 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8870000000000004e-06 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084425+0000", 2023-10-09T04:44:10.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2170000000000002e-06 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084483+0000", 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8044999999999997e-05 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.084495+0000", 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1949e-05 2023-10-09T04:44:10.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.789643+0000", 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.70514765800000001 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.789734+0000", 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1048e-05 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.790220+0000", 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00048593799999999999 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804857+0000", 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.014637229 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.804893+0000", 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5939999999999998e-05 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811704+0000", 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068106199999999999 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811745+0000", 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1927e-05 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:41:36.811782+0000", 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6417e-05 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:75905 8.c 8:3c60d28b:::benchmark_data_smithi082_175677_object75904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:25.835042+0000", 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 44.371296002999998, 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.17240556800000001, 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 75905 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.353 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835042+0000", 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835042+0000", 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835040+0000", 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835064+0000", 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3220000000000001e-05 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835077+0000", 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3393999999999999e-05 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835085+0000", 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7130000000000005e-06 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835123+0000", 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8756e-05 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835268+0000", 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000144332 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.835392+0000", 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012430199999999999 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.841822+0000", 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064300779999999997 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.844490+0000", 2023-10-09T04:44:10.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002667881 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007297+0000", 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.16280667600000001 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007385+0000", 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8089000000000003e-05 2023-10-09T04:44:10.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007448+0000", 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2918999999999998e-05 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:75910 8.c 8:3e608cf7:::benchmark_data_smithi082_175677_object75909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:25.838198+0000", 2023-10-09T04:44:10.360 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 44.368139694, 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.16948993100000001, 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 75910 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838198+0000", 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838198+0000", 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838196+0000", 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838227+0000", 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0577999999999998e-05 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838242+0000", 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5264000000000002e-05 2023-10-09T04:44:10.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838252+0000", 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.978e-06 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838779+0000", 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00052672799999999996 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.838925+0000", 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000146005 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.839072+0000", 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014658499999999999 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:44:10.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.842027+0000", 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002955123 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.844526+0000", 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00249979 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007605+0000", 2023-10-09T04:44:10.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.16307892800000001 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007647+0000", 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1968000000000002e-05 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007688+0000", 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0961999999999998e-05 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.367 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:75917 8.9 e225/218 8:9049dd78:::benchmark_data_smithi082_175677_object75916:head v 225'4721, mlcod=225'4719)", 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:25.847751+0000", 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 44.358587587000002, 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.15965774999999999, 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.368 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847751+0000", 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847751+0000", 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847749+0000", 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847788+0000", 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8627000000000001e-05 2023-10-09T04:44:10.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847803+0000", 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4602e-05 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847812+0000", 2023-10-09T04:44:10.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5370000000000003e-06 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847862+0000", 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9784000000000003e-05 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.847884+0000", 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2512999999999999e-05 2023-10-09T04:44:10.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007357+0000", 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.15947281799999999 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007408+0000", 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0939000000000001e-05 2023-10-09T04:44:10.373 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:75926 8.1 e225/218 8:8c6cf6de:::benchmark_data_smithi082_175677_object75925:head v 225'4836, mlcod=225'4834)", 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:43:25.854768+0000", 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 44.351570398, 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.16414701300000001, 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.854768+0000", 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.854768+0000", 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:10.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.854766+0000", 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:25.854795+0000", 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8639000000000001e-05 2023-10-09T04:44:10.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007224+0000", 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.15242926100000001 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007245+0000", 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0948999999999999e-05 2023-10-09T04:44:10.377 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007375+0000", 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012960000000000001 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.007440+0000", 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5782000000000001e-05 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.018860+0000", 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011419250000000001 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:43:26.018915+0000", 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4956e-05 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.379 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:10.380 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:10.380 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:44:10.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.401 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:10 smithi082 bash[77063]: cluster 2023-10-09T04:44:08.583986+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v948: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 42405/339813 objects degraded (12.479%) 2023-10-09T04:44:10.401 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:10 smithi082 bash[77063]: cluster 2023-10-09T04:44:09.923210+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 42405/339813 objects degraded (12.479%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:10.401 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:10 smithi082 bash[79733]: cluster 2023-10-09T04:44:08.583986+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v948: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 42405/339813 objects degraded (12.479%) 2023-10-09T04:44:10.401 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:10 smithi082 bash[79733]: cluster 2023-10-09T04:44:09.923210+0000 mon.a (mon.0) 1336 : cluster 3 Health check update: Degraded data redundancy: 42405/339813 objects degraded (12.479%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:10.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:10.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:10.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.724 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 90.0207 2023-10-09T04:44:10.724 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 117011 2023-10-09T04:44:10.724 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:44:10.724 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:44:10.724 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 81.239 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 32.579 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 200.438 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 43 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 1299 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 521.264 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 3207 2023-10-09T04:44:10.725 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 688 2023-10-09T04:44:10.726 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.0123043 2023-10-09T04:44:10.726 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.0122505 2023-10-09T04:44:10.726 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 0.429926 2023-10-09T04:44:10.726 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00125881 2023-10-09T04:44:10.735 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:44:10.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:10.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:10.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:10.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:10.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:11.010 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:11.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:11.048 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:44:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:11.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:11.153 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:11.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:11.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:11.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:11 smithi082 bash[77063]: audit 2023-10-09T04:44:10.538457+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.82:0/943087156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:11 smithi082 bash[77063]: audit 2023-10-09T04:44:10.538846+0000 mon.a (mon.0) 1337 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:11 smithi082 bash[79733]: audit 2023-10-09T04:44:10.538457+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.82:0/943087156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:11 smithi082 bash[79733]: audit 2023-10-09T04:44:10.538846+0000 mon.a (mon.0) 1337 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:44:11.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:11.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:11.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:11 smithi153 bash[59565]: audit 2023-10-09T04:44:10.538457+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.82:0/943087156' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:11 smithi153 bash[59565]: audit 2023-10-09T04:44:10.538846+0000 mon.a (mon.0) 1337 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:12.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:12.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:12.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:12.363 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:10.585498+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v949: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:12.363 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:11.016629+0000 mon.a (mon.0) 1338 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:12.363 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:11.032491+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:11.033539+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: audit 2023-10-09T04:44:11.036964+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:11.037035+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e230: 8 total, 7 up, 8 in 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[77063]: cluster 2023-10-09T04:44:11.041435+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:10.585498+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v949: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:11.016629+0000 mon.a (mon.0) 1338 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:11.032491+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:11.033539+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: audit 2023-10-09T04:44:11.036964+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:12.364 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:11.037035+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e230: 8 total, 7 up, 8 in 2023-10-09T04:44:12.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:12 smithi082 bash[79733]: cluster 2023-10-09T04:44:11.041435+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:12.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:10.585498+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v949: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:11.016629+0000 mon.a (mon.0) 1338 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:11.032491+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:11.033539+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: audit 2023-10-09T04:44:11.036964+0000 mon.a (mon.0) 1339 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:11.037035+0000 mon.a (mon.0) 1340 : cluster 0 osdmap e230: 8 total, 7 up, 8 in 2023-10-09T04:44:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:12 smithi153 bash[59565]: cluster 2023-10-09T04:44:11.041435+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T04:44:12.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:12.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:12.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:12.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:12.695 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:12.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:12.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:12.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:12.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:12.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:13.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:13.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:13.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:13.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:13.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:13.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:13.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:13.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:13.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:13.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:13.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:13.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:13.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:13.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:13.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:13.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:14.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:14.053 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:14.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.168 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:14 smithi153 bash[59565]: cluster 2023-10-09T04:44:12.586057+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v951: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:14.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:14.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:44:14.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:14.302 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:14 smithi082 bash[77063]: cluster 2023-10-09T04:44:12.586057+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v951: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:14.302 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:14 smithi082 bash[79733]: cluster 2023-10-09T04:44:12.586057+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v951: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 43904/349899 objects degraded (12.548%) 2023-10-09T04:44:14.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:14.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:14.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:14.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:14.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:14.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.713 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:14.743 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:14.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:14.845 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:14.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:14.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:15.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:15.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:15.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:15.229 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:15 smithi153 bash[59565]: audit 2023-10-09T04:44:14.283927+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:15.229 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:15 smithi153 bash[59565]: cluster 2023-10-09T04:44:14.924634+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 43904/349899 objects degraded (12.548%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:15.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:15.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.409 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:15.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:15 smithi082 bash[77063]: audit 2023-10-09T04:44:14.283927+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:15 smithi082 bash[77063]: cluster 2023-10-09T04:44:14.924634+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 43904/349899 objects degraded (12.548%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:15 smithi082 bash[79733]: audit 2023-10-09T04:44:14.283927+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:15 smithi082 bash[79733]: cluster 2023-10-09T04:44:14.924634+0000 mon.a (mon.0) 1342 : cluster 3 Health check update: Degraded data redundancy: 43904/349899 objects degraded (12.548%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:15.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:15.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:15.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:15.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:15.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:15.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:15.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:15.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:15.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:15.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:16.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:16.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:16.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:16.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:16 smithi082 bash[77063]: cluster 2023-10-09T04:44:14.586795+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v952: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 762 op/s; 43270/345708 objects degraded (12.516%) 2023-10-09T04:44:16.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:16 smithi082 bash[79733]: cluster 2023-10-09T04:44:14.586795+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v952: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 762 op/s; 43270/345708 objects degraded (12.516%) 2023-10-09T04:44:16.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:16.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:16.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:16 smithi153 bash[59565]: cluster 2023-10-09T04:44:14.586795+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v952: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 762 op/s; 43270/345708 objects degraded (12.516%) 2023-10-09T04:44:16.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:16.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:16.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:16.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:16.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:16.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:16.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:16.785 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:16.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:16.898 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:16.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:17.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:17.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:17.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:17.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:17.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:17.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:17.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:17.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:17.322 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:17 smithi082 bash[77063]: cluster 2023-10-09T04:44:16.589886+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v953: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 747 op/s; 42800/341415 objects degraded (12.536%) 2023-10-09T04:44:17.322 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:17 smithi082 bash[79733]: cluster 2023-10-09T04:44:16.589886+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v953: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 747 op/s; 42800/341415 objects degraded (12.536%) 2023-10-09T04:44:17.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:17.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:17 smithi153 bash[59565]: cluster 2023-10-09T04:44:16.589886+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v953: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 747 op/s; 42800/341415 objects degraded (12.536%) 2023-10-09T04:44:17.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:17.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:17.566 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:17.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:17.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:17.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:17.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:17.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:17.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:17.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:17.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:18.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:18.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:18.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:18.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:18.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:18.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:18.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:18.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:18.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:18.802 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:18.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:18.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:18.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:19.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:19.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:19.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:19.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:19.269 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.295 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:19.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:19.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:19.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:19.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:19 smithi153 bash[59565]: cluster 2023-10-09T04:44:18.590426+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v954: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 755 op/s; 42800/339753 objects degraded (12.597%) 2023-10-09T04:44:19.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:19.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:19 smithi082 bash[77063]: cluster 2023-10-09T04:44:18.590426+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v954: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 755 op/s; 42800/339753 objects degraded (12.597%) 2023-10-09T04:44:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:19 smithi082 bash[79733]: cluster 2023-10-09T04:44:18.590426+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v954: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 755 op/s; 42800/339753 objects degraded (12.597%) 2023-10-09T04:44:19.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:19] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T04:44:19.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:19.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:19.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:19.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:19.968 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:19.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:20.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:20.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:20.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:20.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:20.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:20.493 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:20.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:20.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:20.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:20 smithi082 bash[77063]: cluster 2023-10-09T04:44:19.925670+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 42800/339753 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:20.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:20 smithi082 bash[79733]: cluster 2023-10-09T04:44:19.925670+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 42800/339753 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:20.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:20.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:20.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:20.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:20 smithi153 bash[59565]: cluster 2023-10-09T04:44:19.925670+0000 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 42800/339753 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:20.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:21.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:21.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:21.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.271 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:21.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:21.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:21.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:21.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:21.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:21.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:21.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:21.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:21.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:22.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:22.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:22.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.185 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:21 smithi082 bash[77063]: cluster 2023-10-09T04:44:20.591094+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v955: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.5 MiB/s wr, 788 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:22.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:21 smithi082 bash[79733]: cluster 2023-10-09T04:44:20.591094+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v955: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.5 MiB/s wr, 788 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:22.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:22.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:22.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:21 smithi153 bash[59565]: cluster 2023-10-09T04:44:20.591094+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v955: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 2.5 MiB/s wr, 788 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:22.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:22.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:22.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:22.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:22.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:22.713 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:22.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:22.866 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:22.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:22.990 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:22.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:23.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:23.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:23.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:23.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:23.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:23.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:23.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:23.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:23.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:23.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:23.665 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:23.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:23.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:23.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:23.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:23.962 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.1 primary-affinity to 0.926338 (8607082) 2023-10-09T04:44:23.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:23.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:24.086 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:24.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:23 smithi153 bash[59565]: cluster 2023-10-09T04:44:22.592151+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v956: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 88 B/s rd, 2.2 MiB/s wr, 682 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:24.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:23 smithi153 bash[59565]: audit 2023-10-09T04:44:23.494292+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.82:0/2824049952' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:23 smithi153 bash[59565]: audit 2023-10-09T04:44:23.494672+0000 mon.a (mon.0) 1344 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:24.219 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[77063]: cluster 2023-10-09T04:44:22.592151+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v956: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 88 B/s rd, 2.2 MiB/s wr, 682 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:24.220 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[77063]: audit 2023-10-09T04:44:23.494292+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.82:0/2824049952' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.220 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[77063]: audit 2023-10-09T04:44:23.494672+0000 mon.a (mon.0) 1344 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.220 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[79733]: cluster 2023-10-09T04:44:22.592151+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v956: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 88 B/s rd, 2.2 MiB/s wr, 682 op/s; 41216/328677 objects degraded (12.540%) 2023-10-09T04:44:24.220 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[79733]: audit 2023-10-09T04:44:23.494292+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.82:0/2824049952' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.220 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:23 smithi082 bash[79733]: audit 2023-10-09T04:44:23.494672+0000 mon.a (mon.0) 1344 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]: dispatch 2023-10-09T04:44:24.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:24.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:24.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:24.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:24.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:24.521 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:24.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:24.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:24.754 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:24.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:24.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:24.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:24.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:25.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:25.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:25.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:25.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:24 smithi153 bash[59565]: cluster 2023-10-09T04:44:23.935945+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:24 smithi153 bash[59565]: cluster 2023-10-09T04:44:23.937141+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:24 smithi153 bash[59565]: audit 2023-10-09T04:44:23.943748+0000 mon.a (mon.0) 1345 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]': finished 2023-10-09T04:44:25.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:24 smithi153 bash[59565]: cluster 2023-10-09T04:44:23.943848+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e231: 8 total, 7 up, 8 in 2023-10-09T04:44:25.169 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:24 smithi153 bash[59565]: cluster 2023-10-09T04:44:24.926903+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 41216/328677 objects degraded (12.540%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:25.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[77063]: cluster 2023-10-09T04:44:23.935945+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[77063]: cluster 2023-10-09T04:44:23.937141+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[77063]: audit 2023-10-09T04:44:23.943748+0000 mon.a (mon.0) 1345 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]': finished 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[77063]: cluster 2023-10-09T04:44:23.943848+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e231: 8 total, 7 up, 8 in 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[77063]: cluster 2023-10-09T04:44:24.926903+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 41216/328677 objects degraded (12.540%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[79733]: cluster 2023-10-09T04:44:23.935945+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[79733]: cluster 2023-10-09T04:44:23.937141+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[79733]: audit 2023-10-09T04:44:23.943748+0000 mon.a (mon.0) 1345 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9263379889292654}]': finished 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[79733]: cluster 2023-10-09T04:44:23.943848+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e231: 8 total, 7 up, 8 in 2023-10-09T04:44:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:24 smithi082 bash[79733]: cluster 2023-10-09T04:44:24.926903+0000 mon.a (mon.0) 1347 : cluster 3 Health check update: Degraded data redundancy: 41216/328677 objects degraded (12.540%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:25.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:25.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:25.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:25.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:25.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:25.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:25.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:25.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:25.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:25.882 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:25.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:25.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:26.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:26.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:26.124 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:25 smithi153 bash[59565]: cluster 2023-10-09T04:44:24.593791+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v958: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 827 op/s; 40170/320883 objects degraded (12.519%) 2023-10-09T04:44:26.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:26.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:25 smithi082 bash[77063]: cluster 2023-10-09T04:44:24.593791+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v958: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 827 op/s; 40170/320883 objects degraded (12.519%) 2023-10-09T04:44:26.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:25 smithi082 bash[79733]: cluster 2023-10-09T04:44:24.593791+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v958: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 827 op/s; 40170/320883 objects degraded (12.519%) 2023-10-09T04:44:26.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:26.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:26.400 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:26.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.527 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:26.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:26.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:26.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:26.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:26.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:26.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:27.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:27.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:27.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:27.184 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:27.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:27.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:27.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:27.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.574 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:27.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:27.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:27.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:27.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:27.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:27.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:27.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:28.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:28.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:28.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:27 smithi082 bash[77063]: cluster 2023-10-09T04:44:26.594570+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v959: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 39659/316374 objects degraded (12.535%) 2023-10-09T04:44:28.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:27 smithi082 bash[79733]: cluster 2023-10-09T04:44:26.594570+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v959: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 39659/316374 objects degraded (12.535%) 2023-10-09T04:44:28.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:28.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:28.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:27 smithi153 bash[59565]: cluster 2023-10-09T04:44:26.594570+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v959: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 39659/316374 objects degraded (12.535%) 2023-10-09T04:44:28.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:28.498 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:28.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:28.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:28.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:28.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:28.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:28.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:28.924 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:28.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:28.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:29.053 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:29.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:29.193 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:29.235 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:29.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:29.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:29.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:29.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:29.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:29.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:29.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:29.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:29.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:29.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:29.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:29.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:29.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:29.979 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[77063]: cluster 2023-10-09T04:44:28.595312+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v960: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 39659/314820 objects degraded (12.597%) 2023-10-09T04:44:29.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[77063]: audit 2023-10-09T04:44:29.284186+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:29.980 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[77063]: cluster 2023-10-09T04:44:29.927803+0000 mon.a (mon.0) 1349 : cluster 3 Health check update: Degraded data redundancy: 39659/314820 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:29.980 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:29] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:44:30.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:30.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:30.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:30.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:30.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[79733]: cluster 2023-10-09T04:44:28.595312+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v960: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 39659/314820 objects degraded (12.597%) 2023-10-09T04:44:30.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[79733]: audit 2023-10-09T04:44:29.284186+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:30.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:29 smithi082 bash[79733]: cluster 2023-10-09T04:44:29.927803+0000 mon.a (mon.0) 1349 : cluster 3 Health check update: Degraded data redundancy: 39659/314820 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:30.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:30.293 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:29 smithi153 bash[59565]: cluster 2023-10-09T04:44:28.595312+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v960: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 39659/314820 objects degraded (12.597%) 2023-10-09T04:44:30.294 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:29 smithi153 bash[59565]: audit 2023-10-09T04:44:29.284186+0000 mon.a (mon.0) 1348 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:30.294 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:29 smithi153 bash[59565]: cluster 2023-10-09T04:44:29.927803+0000 mon.a (mon.0) 1349 : cluster 3 Health check update: Degraded data redundancy: 39659/314820 objects degraded (12.597%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:30.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:30.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:30.469 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:30.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:30.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:30.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:30.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:30.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:30.750 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:44:30.781 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:44:30.808 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:44:30.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:30.847 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:44:30.867 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:44:30.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:30.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:30.893 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:44:30.919 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:44:30.980 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:31.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:31.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:31.125 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:30 smithi153 bash[59565]: audit 2023-10-09T04:44:30.703515+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.82:0/1329519217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:31.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:31.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:31.258 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:30 smithi082 bash[77063]: audit 2023-10-09T04:44:30.703515+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.82:0/1329519217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:31.259 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:30 smithi082 bash[79733]: audit 2023-10-09T04:44:30.703515+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.82:0/1329519217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:31.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:31.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:31.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:31.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:31.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:31.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:31.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:31.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:31.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:31.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:31.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:31.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:32.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:32.081 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:32.187 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:32.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:32.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:32.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:32 smithi082 bash[77063]: cluster 2023-10-09T04:44:30.596128+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v961: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:32.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:32 smithi082 bash[79733]: cluster 2023-10-09T04:44:30.596128+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v961: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:32.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:32.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:32.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:31 smithi153 bash[59565]: cluster 2023-10-09T04:44:30.596128+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v961: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:32.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:32.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:32.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:32.635 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:32.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:32.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:32.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:32.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:32.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:32.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:33.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:33.045 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:33.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.169 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:44:33.169 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:44:33.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:33.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:33.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:33.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:33.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:33.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:33.539 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:33.669 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:44:33.675 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:44:33.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:33.802 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:33.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:33.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.970 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:33.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:34.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:34.090 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:33 smithi153 bash[59565]: cluster 2023-10-09T04:44:32.596698+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v962: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:34.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:34.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.278 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:34.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:34.381 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:34.391 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:33 smithi082 bash[77063]: cluster 2023-10-09T04:44:32.596698+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v962: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:34.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:33 smithi082 bash[79733]: cluster 2023-10-09T04:44:32.596698+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v962: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 38045/303927 objects degraded (12.518%) 2023-10-09T04:44:34.412 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:34.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:34.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:34.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:34.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:44:34.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:34.725 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:34.833 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:34.844 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 7, 6, 5] 2023-10-09T04:44:34.844 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-10-09T04:44:34.845 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-10-09T04:44:34.845 DEBUG:teuthology.orchestra.run.smithi082:osd.2> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T04:44:34.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:34.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl start ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T04:44:34.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:34.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:34.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:44:34.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:34.993 INFO:journalctl@ceph.osd.2.smithi082.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:44:35.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:35.034 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:44:35.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.172 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:35.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:35.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:35 smithi153 bash[59565]: audit 2023-10-09T04:44:34.782982+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.82:0/582137333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:35 smithi153 bash[59565]: audit 2023-10-09T04:44:34.783956+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.284 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:35 smithi153 bash[59565]: cluster 2023-10-09T04:44:34.928933+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 38045/303927 objects degraded (12.518%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:35.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:35.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[77063]: audit 2023-10-09T04:44:34.782982+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.82:0/582137333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.397 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[77063]: audit 2023-10-09T04:44:34.783956+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.397 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[77063]: cluster 2023-10-09T04:44:34.928933+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 38045/303927 objects degraded (12.518%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:35.397 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[79733]: audit 2023-10-09T04:44:34.782982+0000 mon.c (mon.2) 99 : audit 1 from='client.? 172.21.15.82:0/582137333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.397 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[79733]: audit 2023-10-09T04:44:34.783956+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:35.397 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:35 smithi082 bash[79733]: cluster 2023-10-09T04:44:34.928933+0000 mon.a (mon.0) 1351 : cluster 3 Health check update: Degraded data redundancy: 38045/303927 objects degraded (12.518%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:35.405 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:35.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:35.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:35.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:44:35.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:35.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:35.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:35.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:35.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:35.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:36.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:36.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:36.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:34.597617+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v963: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 37043/296508 objects degraded (12.493%) 2023-10-09T04:44:36.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:34.997148+0000 mon.a (mon.0) 1352 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:36.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:35.015668+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:35.017756+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:35.020895+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: audit 2023-10-09T04:44:35.022013+0000 mon.a (mon.0) 1353 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:36.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:35.022049+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e232: 8 total, 7 up, 8 in 2023-10-09T04:44:36.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:36 smithi153 bash[59565]: cluster 2023-10-09T04:44:35.024411+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.213 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:36.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:36.271 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:34.597617+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v963: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 37043/296508 objects degraded (12.493%) 2023-10-09T04:44:36.271 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:34.997148+0000 mon.a (mon.0) 1352 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:35.015668+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:35.017756+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:35.020895+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: audit 2023-10-09T04:44:35.022013+0000 mon.a (mon.0) 1353 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:35.022049+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e232: 8 total, 7 up, 8 in 2023-10-09T04:44:36.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[79733]: cluster 2023-10-09T04:44:35.024411+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:34.597617+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v963: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 37043/296508 objects degraded (12.493%) 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:34.997148+0000 mon.a (mon.0) 1352 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:35.015668+0000 osd.4 (osd.4) 179 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:35.017756+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:35.020895+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: audit 2023-10-09T04:44:35.022013+0000 mon.a (mon.0) 1353 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:35.022049+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e232: 8 total, 7 up, 8 in 2023-10-09T04:44:36.273 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:36 smithi082 bash[77063]: cluster 2023-10-09T04:44:35.024411+0000 osd.4 (osd.4) 180 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T04:44:36.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:36.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:36.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.644 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:36.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:36.777 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:36.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:36.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:36.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:37.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:37.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:37.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:37.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:37.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:37.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:37.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:37.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:37.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:37.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:37.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:37.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:37.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:37.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:37.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:37.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:37.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:38.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:38.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:38.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:38.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:38.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[77063]: cluster 2023-10-09T04:44:36.598411+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v965: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 36520/292140 objects degraded (12.501%) 2023-10-09T04:44:38.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[79733]: cluster 2023-10-09T04:44:36.598411+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v965: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 36520/292140 objects degraded (12.501%) 2023-10-09T04:44:38.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:38.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:38.410 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:38.419 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:38 smithi153 bash[59565]: cluster 2023-10-09T04:44:36.598411+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v965: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 36520/292140 objects degraded (12.501%) 2023-10-09T04:44:38.439 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:38.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:38.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:38.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:38.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:38.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:38.836 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:44:38.836 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:44:38.837 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:38 smithi082 bash[183199]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T04:44:38.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:38.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:38.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:39.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:39.142 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:39.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:39.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:39.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:44:39.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:39.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.545 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:39.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:39.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:39.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:39] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:44:39.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:39.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:39.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:39.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:40.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:40.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:40.199 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:40.229 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:40 smithi082 bash[77063]: cluster 2023-10-09T04:44:38.599159+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v966: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 36520/290838 objects degraded (12.557%) 2023-10-09T04:44:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:40 smithi082 bash[77063]: cluster 2023-10-09T04:44:39.930279+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 36520/290838 objects degraded (12.557%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:40.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:40 smithi082 bash[79733]: cluster 2023-10-09T04:44:38.599159+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v966: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 36520/290838 objects degraded (12.557%) 2023-10-09T04:44:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:40 smithi082 bash[79733]: cluster 2023-10-09T04:44:39.930279+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 36520/290838 objects degraded (12.557%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:40.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:40.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:40 smithi153 bash[59565]: cluster 2023-10-09T04:44:38.599159+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v966: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 36520/290838 objects degraded (12.557%) 2023-10-09T04:44:40.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:40 smithi153 bash[59565]: cluster 2023-10-09T04:44:39.930279+0000 mon.a (mon.0) 1355 : cluster 3 Health check update: Degraded data redundancy: 36520/290838 objects degraded (12.557%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:40.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:40.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:40.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:40.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:40.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:40.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:40.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:40.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:40.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:40.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:41.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:41.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:41.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:41.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.316 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:41.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:41.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:41.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:41.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:41.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:41.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:41.931 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:41 smithi082 bash[183703]: debug 2023-10-09T04:44:41.678+0000 7f6901a43700 1 -- 172.21.15.82:0/1513527485 <== mon.2 v2:172.21.15.82:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5595b70884e0 con 0x5595b70e2800 2023-10-09T04:44:41.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:41.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:42.036 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:42 smithi153 bash[59565]: cluster 2023-10-09T04:44:40.600059+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v967: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:42.036 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:42 smithi153 bash[59565]: audit 2023-10-09T04:44:41.246190+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:42.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:42.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:42.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:42 smithi082 bash[77063]: cluster 2023-10-09T04:44:40.600059+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v967: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:42.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:42 smithi082 bash[77063]: audit 2023-10-09T04:44:41.246190+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:42.211 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:42 smithi082 bash[79733]: cluster 2023-10-09T04:44:40.600059+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v967: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:42.212 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:42 smithi082 bash[79733]: audit 2023-10-09T04:44:41.246190+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:42.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:42.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:42.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:42.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:42.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:42.477 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:42 smithi082 bash[183703]: debug 2023-10-09T04:44:42.254+0000 7f690abc9640 -1 Falling back to public interface 2023-10-09T04:44:42.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:42.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:42.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:42.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:42.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:42.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:42.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:42.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:43.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:43.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:43.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:43.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:43.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:43.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:43.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:43.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:43.563 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:43.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:43.566 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:43.566 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:43.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:43.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:43.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:43.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:43.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:43.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:43.965 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:44.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:44.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:44.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:44 smithi153 bash[59565]: cluster 2023-10-09T04:44:42.600709+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v968: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:44.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:44.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:44.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:44.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:44.398 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:44 smithi082 bash[77063]: cluster 2023-10-09T04:44:42.600709+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v968: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:44.398 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:44 smithi082 bash[79733]: cluster 2023-10-09T04:44:42.600709+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v968: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 35083/281085 objects degraded (12.481%) 2023-10-09T04:44:44.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:44.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:44.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:44.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:44.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:44.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:44.786 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:44.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:44.931 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:44.968 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:44.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:45.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:45.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:45.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:45.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:45.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:45.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:45.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[77063]: audit 2023-10-09T04:44:44.290241+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[77063]: audit 2023-10-09T04:44:44.291632+0000 mon.a (mon.0) 1358 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[77063]: cluster 2023-10-09T04:44:44.601488+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v969: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 34095/273666 objects degraded (12.459%) 2023-10-09T04:44:45.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[77063]: cluster 2023-10-09T04:44:44.931363+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 35083/281085 objects degraded (12.481%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[79733]: audit 2023-10-09T04:44:44.290241+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[79733]: audit 2023-10-09T04:44:44.291632+0000 mon.a (mon.0) 1358 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[79733]: cluster 2023-10-09T04:44:44.601488+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v969: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 34095/273666 objects degraded (12.459%) 2023-10-09T04:44:45.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:45 smithi082 bash[79733]: cluster 2023-10-09T04:44:44.931363+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 35083/281085 objects degraded (12.481%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:45.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:45.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:45.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:45 smithi153 bash[59565]: audit 2023-10-09T04:44:44.290241+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:45.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:45 smithi153 bash[59565]: audit 2023-10-09T04:44:44.291632+0000 mon.a (mon.0) 1358 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:45.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:45 smithi153 bash[59565]: cluster 2023-10-09T04:44:44.601488+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v969: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 34095/273666 objects degraded (12.459%) 2023-10-09T04:44:45.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:45 smithi153 bash[59565]: cluster 2023-10-09T04:44:44.931363+0000 mon.a (mon.0) 1359 : cluster 3 Health check update: Degraded data redundancy: 35083/281085 objects degraded (12.481%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:45.609 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:45.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:45.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:45.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:45.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:45.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:45.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:45.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:46.026 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:46.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:46.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:46.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:46.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:46.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:46.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:46.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:46.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.643 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:46.748 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.783 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:46.891 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:46.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:46.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:47.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:47.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:47.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:47.164 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:47.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:47.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:47.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:47.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:47.473 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:47.578 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:47.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:47.752 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:47.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.865 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:47 smithi082 bash[77063]: cluster 2023-10-09T04:44:46.602369+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v970: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 33604/269241 objects degraded (12.481%) 2023-10-09T04:44:47.865 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:47 smithi082 bash[79733]: cluster 2023-10-09T04:44:46.602369+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v970: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 33604/269241 objects degraded (12.481%) 2023-10-09T04:44:47.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:47.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:47 smithi153 bash[59565]: cluster 2023-10-09T04:44:46.602369+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v970: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 33604/269241 objects degraded (12.481%) 2023-10-09T04:44:47.987 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:48.012 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:48.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:48.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:48.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:48.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:48.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:48.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:48.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:48.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:48.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:48.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:48.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:48.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:48.949 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:48.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:48.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:49.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:49.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:47.850130+0000 mon.a (mon.0) 1360 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:47.863632+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:47.874378+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.094 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:47.888602+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:48.577913+0000 mon.a (mon.0) 1364 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:49.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:48.579135+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:49.095 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:48 smithi153 bash[59565]: audit 2023-10-09T04:44:48.596774+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.113 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:49.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:47.850130+0000 mon.a (mon.0) 1360 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:47.863632+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:47.874378+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:47.888602+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:48.577913+0000 mon.a (mon.0) 1364 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:48.579135+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[77063]: audit 2023-10-09T04:44:48.596774+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:47.850130+0000 mon.a (mon.0) 1360 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:47.863632+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:47.874378+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:47.888602+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.158 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:48.577913+0000 mon.a (mon.0) 1364 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:44:49.158 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:48.579135+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:44:49.158 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:48 smithi082 bash[79733]: audit 2023-10-09T04:44:48.596774+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:44:49.158 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[183703]: debug 2023-10-09T04:44:49.142+0000 7f690abc9640 -1 osd.2 217 log_to_monitors true 2023-10-09T04:44:49.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:49.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:49.355 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:49.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:49.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:49.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:49.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:49.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:49.643 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:49.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:49.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:49.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:49.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:49.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:49] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T04:44:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[77063]: cluster 2023-10-09T04:44:48.603088+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v971: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 33604/267825 objects degraded (12.547%) 2023-10-09T04:44:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[77063]: audit 2023-10-09T04:44:49.157140+0000 mon.c (mon.2) 100 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[77063]: audit 2023-10-09T04:44:49.157883+0000 mon.a (mon.0) 1367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[79733]: cluster 2023-10-09T04:44:48.603088+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v971: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 33604/267825 objects degraded (12.547%) 2023-10-09T04:44:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[79733]: audit 2023-10-09T04:44:49.157140+0000 mon.c (mon.2) 100 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[79733]: audit 2023-10-09T04:44:49.157883+0000 mon.a (mon.0) 1367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:49.983 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 04:44:49 smithi082 bash[183703]: debug 2023-10-09T04:44:49.930+0000 7f68f802d700 -1 osd.2 217 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:44:49.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:50.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:49 smithi153 bash[59565]: cluster 2023-10-09T04:44:48.603088+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v971: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 33604/267825 objects degraded (12.547%) 2023-10-09T04:44:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:49 smithi153 bash[59565]: audit 2023-10-09T04:44:49.157140+0000 mon.c (mon.2) 100 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:50.149 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:49 smithi153 bash[59565]: audit 2023-10-09T04:44:49.157883+0000 mon.a (mon.0) 1367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:44:50.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:50.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:50.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:50.428 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:50.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.573 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:50.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:50.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:50.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:50.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:50.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:50.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:50.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:51.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: audit 2023-10-09T04:44:49.864918+0000 mon.a (mon.0) 1368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.865052+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e233: 8 total, 7 up, 8 in 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.868464+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.870343+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.872997+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.080 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: audit 2023-10-09T04:44:49.873923+0000 mon.c (mon.2) 101 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.875295+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: audit 2023-10-09T04:44:49.876819+0000 mon.a (mon.0) 1370 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[77063]: cluster 2023-10-09T04:44:49.932734+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 33604/267825 objects degraded (12.547%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: audit 2023-10-09T04:44:49.864918+0000 mon.a (mon.0) 1368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.865052+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e233: 8 total, 7 up, 8 in 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.868464+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.081 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.870343+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.872997+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: audit 2023-10-09T04:44:49.873923+0000 mon.c (mon.2) 101 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.875295+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: audit 2023-10-09T04:44:49.876819+0000 mon.a (mon.0) 1370 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:50 smithi082 bash[79733]: cluster 2023-10-09T04:44:49.932734+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 33604/267825 objects degraded (12.547%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:51.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:51.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: audit 2023-10-09T04:44:49.864918+0000 mon.a (mon.0) 1368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:44:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.865052+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e233: 8 total, 7 up, 8 in 2023-10-09T04:44:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.868464+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.870343+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.872997+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: audit 2023-10-09T04:44:49.873923+0000 mon.c (mon.2) 101 : audit 1 from='osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.875295+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T04:44:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: audit 2023-10-09T04:44:49.876819+0000 mon.a (mon.0) 1370 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2023-10-09T04:44:51.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:50 smithi153 bash[59565]: cluster 2023-10-09T04:44:49.932734+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 33604/267825 objects degraded (12.547%), 9 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:51.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:51.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:51.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:51.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:51.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:51.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:51.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:51.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:51.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:51.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:51.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:51.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:51.852 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:51.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:51.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:51.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:51.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:52.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:52.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:52.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.603784+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v973: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:52.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.862187+0000 mon.a (mon.0) 1372 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:44:52.143 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.867398+0000 mon.a (mon.0) 1373 : cluster 1 osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668] boot 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.867439+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: audit 2023-10-09T04:44:50.867964+0000 mon.a (mon.0) 1375 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.868743+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.870784+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.871058+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.871849+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.872681+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[77063]: cluster 2023-10-09T04:44:50.881672+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.603784+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v973: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.862187+0000 mon.a (mon.0) 1372 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:44:52.144 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.867398+0000 mon.a (mon.0) 1373 : cluster 1 osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668] boot 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.867439+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: audit 2023-10-09T04:44:50.867964+0000 mon.a (mon.0) 1375 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.868743+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.870784+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.871058+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.871849+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.872681+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.146 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:51 smithi082 bash[79733]: cluster 2023-10-09T04:44:50.881672+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.603784+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v973: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.862187+0000 mon.a (mon.0) 1372 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.867398+0000 mon.a (mon.0) 1373 : cluster 1 osd.2 [v2:172.21.15.82:6818/2157929668,v1:172.21.15.82:6819/2157929668] boot 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.867439+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: audit 2023-10-09T04:44:50.867964+0000 mon.a (mon.0) 1375 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.868743+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.870784+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.871058+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.871849+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.872681+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:51 smithi153 bash[59565]: cluster 2023-10-09T04:44:50.881672+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T04:44:52.265 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:52.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:52.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:52.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:52.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:52.567 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:52.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:52.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:52.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:52.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:52.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:52.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:52.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:52.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:53.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:53.076 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.873592+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.076 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.875384+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.879034+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.879222+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.879846+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.880291+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.882370+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.883757+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:51.889229+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.077 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.877033+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.873592+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.875384+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.879034+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.879222+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.879846+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.880291+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.882370+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.883757+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:51.889229+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.078 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:52 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.877033+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:44:53.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:53.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.873592+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.875384+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.879034+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.879222+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.879846+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.880291+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.882370+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.883757+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:51.889229+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T04:44:53.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:52 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.877033+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:44:53.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.230 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:53.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:53.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:53.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:53.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:53.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:53.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.765 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:53.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:53.918 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:53.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:53.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:54.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.604788+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v976: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.875660+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.877682+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.878573+0000 osd.1 (osd.1) 5 : cluster 0 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 233'9426 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.879655+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.880175+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.882016+0000 osd.1 (osd.1) 6 : cluster 0 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 233'9394 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.882220+0000 osd.1 (osd.1) 7 : cluster 0 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 233'9279 2023-10-09T04:44:54.034 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:53 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.884243+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:54.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:54.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.604788+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v976: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.875660+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.877682+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.878573+0000 osd.1 (osd.1) 5 : cluster 0 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 233'9426 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.879655+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.880175+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.882016+0000 osd.1 (osd.1) 6 : cluster 0 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 233'9394 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.882220+0000 osd.1 (osd.1) 7 : cluster 0 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 233'9279 2023-10-09T04:44:54.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.884243+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.604788+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v976: 89 pgs: 2 active+clean+remapped, 9 active+undersized+degraded, 14 active+undersized, 64 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 32262/258540 objects degraded (12.479%) 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.875660+0000 osd.5 (osd.5) 140 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.877682+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.878573+0000 osd.1 (osd.1) 5 : cluster 0 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 233'9426 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.879655+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.880175+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.882016+0000 osd.1 (osd.1) 6 : cluster 0 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 233'9394 2023-10-09T04:44:54.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.882220+0000 osd.1 (osd.1) 7 : cluster 0 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 233'9279 2023-10-09T04:44:54.172 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:53 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.884243+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:54.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:54.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:54.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:54.429 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:54.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:54.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:54.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:54.698 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:54.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:54.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:54.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:54.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:54.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:54.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:55.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:55.109 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:54 smithi153 bash[59565]: cluster 2023-10-09T04:44:52.987218+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:55.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:54 smithi082 bash[77063]: cluster 2023-10-09T04:44:52.987218+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:55.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:54 smithi082 bash[79733]: cluster 2023-10-09T04:44:52.987218+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T04:44:55.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:55.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:55.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:55.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:55.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:55.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:55.530 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:55.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:55.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:55.772 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:55.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:55.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:55.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:55.915 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:55 smithi082 bash[77063]: cluster 2023-10-09T04:44:54.605970+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v978: 89 pgs: 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 73 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 31756/255120 objects degraded (12.447%); 877 KiB/s, 13 objects/s recovering 2023-10-09T04:44:55.915 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:55 smithi082 bash[77063]: cluster 2023-10-09T04:44:54.936906+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 31756/255120 objects degraded (12.447%), 8 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:55.931 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:55.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:55.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:56.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:56.045 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:55 smithi153 bash[59565]: cluster 2023-10-09T04:44:54.605970+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v978: 89 pgs: 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 73 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 31756/255120 objects degraded (12.447%); 877 KiB/s, 13 objects/s recovering 2023-10-09T04:44:56.046 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:55 smithi153 bash[59565]: cluster 2023-10-09T04:44:54.936906+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 31756/255120 objects degraded (12.447%), 8 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:56.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:56.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:56.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:55 smithi082 bash[79733]: cluster 2023-10-09T04:44:54.605970+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v978: 89 pgs: 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 73 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 31756/255120 objects degraded (12.447%); 877 KiB/s, 13 objects/s recovering 2023-10-09T04:44:56.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:55 smithi082 bash[79733]: cluster 2023-10-09T04:44:54.936906+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 31756/255120 objects degraded (12.447%), 8 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T04:44:56.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:56.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:56.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:56.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:56.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:56.720 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.754 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:56.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:56.898 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:56.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:57.002 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:57.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.168 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:57.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:57.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:57.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:57.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:57.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:57.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:57.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:57.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:58.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.090 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:57 smithi153 bash[59565]: cluster 2023-10-09T04:44:56.606791+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v979: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 35679/253305 objects degraded (14.085%); 693 KiB/s, 11 objects/s recovering 2023-10-09T04:44:58.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:58.127 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:57 smithi082 bash[79733]: cluster 2023-10-09T04:44:56.606791+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v979: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 35679/253305 objects degraded (14.085%); 693 KiB/s, 11 objects/s recovering 2023-10-09T04:44:58.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:57 smithi082 bash[77063]: cluster 2023-10-09T04:44:56.606791+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v979: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 35679/253305 objects degraded (14.085%); 693 KiB/s, 11 objects/s recovering 2023-10-09T04:44:58.142 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:44:58.183 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:44:58.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:58.233 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:44:58.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:58.277 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:44:58.320 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:44:58.358 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:44:58.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:58.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.407 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:44:58.407 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:44:58.407 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:44:58.407 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:44:58.407 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:44:58.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:58.419 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:44:58.456 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.462 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:44:58.509 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.534 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:58.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152053 8.d e236/235 8:b2e5522f:::benchmark_data_smithi082_175677_object35039:head v 236'9610, mlcod=223'2037)", 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.603972+0000", 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.043741231999999998, 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.043775248000000003, 2023-10-09T04:44:58.659 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603972+0000", 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603972+0000", 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.660 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603971+0000", 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603977+0000", 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9009999999999999e-06 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.661 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603991+0000", 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4483e-05 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.603998+0000", 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9419999999999998e-06 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.662 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.616142+0000", 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012143297000000001 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.616161+0000", 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9754000000000001e-05 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.663 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152055 8.d e236/235 8:bc09f48d:::benchmark_data_smithi082_175677_object35041:head v 236'9611, mlcod=223'2037)", 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.615975+0000", 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.031737697000000002, 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.036106581999999998, 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.615975+0000", 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.664 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.615975+0000", 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.615974+0000", 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.665 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.615985+0000", 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.095e-05 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.616002+0000", 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7405e-05 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.666 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.616013+0000", 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0220999999999999e-05 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.629041+0000", 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013028582 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.629102+0000", 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1184000000000002e-05 2023-10-09T04:44:58.667 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152065 8.e e236/235 8:7a885c6b:::benchmark_data_smithi082_175677_object35051:head v 236'9579, mlcod=223'2198)", 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.627339+0000", 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.020373518, 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.024814574999999998, 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.668 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627339+0000", 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627339+0000", 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.669 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627339+0000", 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627347+0000", 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6470000000000004e-06 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.670 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627358+0000", 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1087000000000001e-05 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627366+0000", 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0899999999999999e-06 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.673 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627659+0000", 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00029339699999999998 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.627683+0000", 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4477999999999998e-05 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.674 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152071 8.e e236/235 8:77c7be1c:::benchmark_data_smithi082_175677_object35057:head v 236'9580, mlcod=223'2198)", 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.634763+0000", 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.012949966, 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017451863000000001, 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.675 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634763+0000", 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634763+0000", 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634762+0000", 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:58.676 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634775+0000", 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3361e-05 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634792+0000", 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6841999999999999e-05 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.677 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.634801+0000", 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9449999999999999e-06 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.637571+0000", 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027704600000000002 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.637598+0000", 2023-10-09T04:44:58.678 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6472e-05 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152075 8.4 e236/235 8:2c7619b3:::benchmark_data_smithi082_175677_object35061:head v 236'9477, mlcod=223'1947)", 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.643231+0000", 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.004481629, 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0090446369999999995, 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.679 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643231+0000", 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643231+0000", 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.680 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643230+0000", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643242+0000", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1833e-05 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643259+0000", 2023-10-09T04:44:58.681 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7557000000000001e-05 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643269+0000", 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5219999999999996e-06 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643456+0000", 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018698200000000001 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.682 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643529+0000", 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3015000000000005e-05 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152076 8.f e236/235 8:fa9b2f7e:::benchmark_data_smithi082_175677_object35062:head v 236'9601, mlcod=236'9599)", 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.643300+0000", 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0044134420000000001, 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0090356359999999997, 2023-10-09T04:44:58.683 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643300+0000", 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643300+0000", 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.684 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643299+0000", 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643304+0000", 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5789999999999997e-06 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.685 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643314+0000", 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0638e-05 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.643319+0000", 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.78e-06 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:44:58.686 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.647789+0000", 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0044697460000000001 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.647815+0000", 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5636999999999999e-05 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.687 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:152050 8.e e236/235)", 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.650496+0000", 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 4294967295.9972172, 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0018952730000000001, 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650496+0000", 2023-10-09T04:44:58.688 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650496+0000", 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650495+0000", 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:44:58.689 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650504+0000", 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5230000000000001e-06 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650530+0000", 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5488000000000001e-05 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:44:58.690 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.650538+0000", 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3340000000000002e-06 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 7 2023-10-09T04:44:58.691 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:44:58.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:58.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:58.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:58.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:58.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:59.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:59.068 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:58 smithi082 bash[77063]: audit 2023-10-09T04:44:58.081384+0000 mon.a (mon.0) 1379 : audit 0 from='client.? 172.21.15.82:0/336687454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:59.068 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:58 smithi082 bash[79733]: audit 2023-10-09T04:44:58.081384+0000 mon.a (mon.0) 1379 : audit 0 from='client.? 172.21.15.82:0/336687454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:59.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:59.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:59.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:58 smithi153 bash[59565]: audit 2023-10-09T04:44:58.081384+0000 mon.a (mon.0) 1379 : audit 0 from='client.? 172.21.15.82:0/336687454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:59.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:59.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:59.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:59.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:59.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:59.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:59.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:59.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:59.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:59.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:59.769 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:59.872 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:59.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:59.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:44:59.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:59.944 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:44:59.944 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:44:59] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:45:00.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:00.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:00.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:00.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:59 smithi153 bash[59565]: cluster 2023-10-09T04:44:58.607364+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v980: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 35679/252309 objects degraded (14.141%); 537 KiB/s, 8 objects/s recovering 2023-10-09T04:45:00.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:59 smithi153 bash[59565]: audit 2023-10-09T04:44:59.024256+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.82:0/2873124954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.154 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:59 smithi153 bash[59565]: audit 2023-10-09T04:44:59.026049+0000 mon.a (mon.0) 1380 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:59 smithi153 bash[59565]: audit 2023-10-09T04:44:59.292293+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:00.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:44:59 smithi153 bash[59565]: audit 2023-10-09T04:44:59.297411+0000 mon.a (mon.0) 1382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:00.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[77063]: cluster 2023-10-09T04:44:58.607364+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v980: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 35679/252309 objects degraded (14.141%); 537 KiB/s, 8 objects/s recovering 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[77063]: audit 2023-10-09T04:44:59.024256+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.82:0/2873124954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[77063]: audit 2023-10-09T04:44:59.026049+0000 mon.a (mon.0) 1380 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[77063]: audit 2023-10-09T04:44:59.292293+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[77063]: audit 2023-10-09T04:44:59.297411+0000 mon.a (mon.0) 1382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[79733]: cluster 2023-10-09T04:44:58.607364+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v980: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 35679/252309 objects degraded (14.141%); 537 KiB/s, 8 objects/s recovering 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[79733]: audit 2023-10-09T04:44:59.024256+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.82:0/2873124954' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[79733]: audit 2023-10-09T04:44:59.026049+0000 mon.a (mon.0) 1380 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:45:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[79733]: audit 2023-10-09T04:44:59.292293+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:00.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:44:59 smithi082 bash[79733]: audit 2023-10-09T04:44:59.297411+0000 mon.a (mon.0) 1382 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:00.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:00.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:00.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:00.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:00.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:00.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:00.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:00.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:00.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:00.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:00.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:00.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:00.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:00.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:01.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:01.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.929534+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.933699+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: audit 2023-10-09T04:44:59.937480+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.937628+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.938338+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.939878+0000 mon.a (mon.0) 1386 : cluster 3 Health check update: Degraded data redundancy: 35679/252309 objects degraded (14.141%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.940654+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.104 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:00 smithi153 bash[59565]: cluster 2023-10-09T04:44:59.940739+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:01.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:01.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:01.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:01.466 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.929534+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:01.466 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.933699+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: audit 2023-10-09T04:44:59.937480+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.937628+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.938338+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.939878+0000 mon.a (mon.0) 1386 : cluster 3 Health check update: Degraded data redundancy: 35679/252309 objects degraded (14.141%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.940654+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[77063]: cluster 2023-10-09T04:44:59.940739+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.929534+0000 mon.a (mon.0) 1383 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.933699+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.467 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: audit 2023-10-09T04:44:59.937480+0000 mon.a (mon.0) 1384 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:45:01.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.937628+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:45:01.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.938338+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.939878+0000 mon.a (mon.0) 1386 : cluster 3 Health check update: Degraded data redundancy: 35679/252309 objects degraded (14.141%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:01.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.940654+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:00 smithi082 bash[79733]: cluster 2023-10-09T04:44:59.940739+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T04:45:01.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:01.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:01.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:01.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:01.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:01.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:01.826 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:01.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:01.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:01.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:01.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:02.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:02.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:01 smithi082 bash[77063]: cluster 2023-10-09T04:45:00.608054+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 35074/244362 objects degraded (14.353%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:02.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:01 smithi082 bash[79733]: cluster 2023-10-09T04:45:00.608054+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 35074/244362 objects degraded (14.353%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:02.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:02.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:45:02.283 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:02.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:02.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:01 smithi153 bash[59565]: cluster 2023-10-09T04:45:00.608054+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 35074/244362 objects degraded (14.353%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:02.411 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:02.514 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:02.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:02.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:02.691 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:02.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:02.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:45:02.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:02.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:02.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:02.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:02.983 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:03.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:03.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:03.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:03.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:03.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:03.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:03.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:03.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:03.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:03.590 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:03.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:03.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:03.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:03.877 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:03.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:03.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:04.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:04.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:04.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:04.125 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:03 smithi153 bash[59565]: cluster 2023-10-09T04:45:02.608669+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 35074/244362 objects degraded (14.353%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:45:04.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:04.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:04.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:03 smithi082 bash[77063]: cluster 2023-10-09T04:45:02.608669+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 35074/244362 objects degraded (14.353%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:45:04.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:03 smithi082 bash[79733]: cluster 2023-10-09T04:45:02.608669+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 35074/244362 objects degraded (14.353%); 848 KiB/s, 13 objects/s recovering 2023-10-09T04:45:04.254 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:04.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:04.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:04.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:04.430 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:04.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:04.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:04.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:04.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:04.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:04.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:45:04.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:45:04.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:04.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:04.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:04.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:05.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:05.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:05.145 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:04 smithi082 bash[77063]: cluster 2023-10-09T04:45:04.941303+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 35074/244362 objects degraded (14.353%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:05.145 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:04 smithi082 bash[79733]: cluster 2023-10-09T04:45:04.941303+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 35074/244362 objects degraded (14.353%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:05.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:05.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:05.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:04 smithi153 bash[59565]: cluster 2023-10-09T04:45:04.941303+0000 mon.a (mon.0) 1387 : cluster 3 Health check update: Degraded data redundancy: 35074/244362 objects degraded (14.353%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:05.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:05.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:05.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:05.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:05.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:05.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:05.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:06.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:06.027 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:05 smithi082 bash[77063]: cluster 2023-10-09T04:45:04.609375+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 34435/238719 objects degraded (14.425%); 832 KiB/s, 13 objects/s recovering 2023-10-09T04:45:06.027 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:05 smithi082 bash[79733]: cluster 2023-10-09T04:45:04.609375+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 34435/238719 objects degraded (14.425%); 832 KiB/s, 13 objects/s recovering 2023-10-09T04:45:06.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:06.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:06.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:05 smithi153 bash[59565]: cluster 2023-10-09T04:45:04.609375+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 34435/238719 objects degraded (14.425%); 832 KiB/s, 13 objects/s recovering 2023-10-09T04:45:06.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:06.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:06.581 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:06.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:06.722 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:06.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:06.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:06.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:06.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:07.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:07.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:07.129 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:07.129 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:07.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:07.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:07.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:07.306 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:07.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:07.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:07.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:07.584 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:07.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:07.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:07.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:07.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:07.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:07.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:08.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.126 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:07 smithi153 bash[59565]: cluster 2023-10-09T04:45:06.610225+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 34435/235506 objects degraded (14.622%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:08.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:08.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:08.268 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:07 smithi082 bash[77063]: cluster 2023-10-09T04:45:06.610225+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 34435/235506 objects degraded (14.622%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:08.269 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:07 smithi082 bash[79733]: cluster 2023-10-09T04:45:06.610225+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 34435/235506 objects degraded (14.622%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:08.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:08.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:08.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:08.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:08.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:08.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:08.693 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:08.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:08.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:08.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:08.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:08.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:09.070 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:09.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:09.167 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:09.168 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:09.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:09.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:09.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:09.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:09.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:09.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:09.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:09.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:09.632 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:09.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:09.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:09.786 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:09] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:45:09.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:09.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:09.942 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:09.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:10.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:10.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:10.173 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:09 smithi082 bash[77063]: cluster 2023-10-09T04:45:08.610756+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34435/234342 objects degraded (14.694%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:09 smithi082 bash[77063]: cluster 2023-10-09T04:45:09.945745+0000 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 34435/234342 objects degraded (14.694%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:10.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:09 smithi082 bash[79733]: cluster 2023-10-09T04:45:08.610756+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34435/234342 objects degraded (14.694%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:10.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:09 smithi082 bash[79733]: cluster 2023-10-09T04:45:09.945745+0000 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 34435/234342 objects degraded (14.694%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:10.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:10.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:09 smithi153 bash[59565]: cluster 2023-10-09T04:45:08.610756+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34435/234342 objects degraded (14.694%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:45:10.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:09 smithi153 bash[59565]: cluster 2023-10-09T04:45:09.945745+0000 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 34435/234342 objects degraded (14.694%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:10.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:10.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:10.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:10.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:10.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:10.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:10.831 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:10.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:10.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:11.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.142 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:11.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:11.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:11.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:11.424 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:11.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:11.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:11.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:11.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:11.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:11.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:11.801 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:11.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:11.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:11.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:11.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:12.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:12.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:12.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:11 smithi153 bash[59565]: cluster 2023-10-09T04:45:10.611500+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 33785/226494 objects degraded (14.917%); 816 KiB/s, 12 objects/s recovering 2023-10-09T04:45:12.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:12.218 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:12.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:11 smithi082 bash[77063]: cluster 2023-10-09T04:45:10.611500+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 33785/226494 objects degraded (14.917%); 816 KiB/s, 12 objects/s recovering 2023-10-09T04:45:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:11 smithi082 bash[79733]: cluster 2023-10-09T04:45:10.611500+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 33785/226494 objects degraded (14.917%); 816 KiB/s, 12 objects/s recovering 2023-10-09T04:45:12.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.364 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:12.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:12.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:12.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:12.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:12.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:12.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:12.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:12.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:13.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:13.057 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:13.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:13.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:13.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:13.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:13.314 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:13.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:13.450 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:13.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:13.573 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:13.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:13.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:13.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:13.901 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:13.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:13.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:13 smithi153 bash[59565]: cluster 2023-10-09T04:45:12.612202+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 33785/226494 objects degraded (14.917%); 725 KiB/s, 11 objects/s recovering 2023-10-09T04:45:14.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:14.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:14.058 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:13 smithi082 bash[77063]: cluster 2023-10-09T04:45:12.612202+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 33785/226494 objects degraded (14.917%); 725 KiB/s, 11 objects/s recovering 2023-10-09T04:45:14.059 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:13 smithi082 bash[79733]: cluster 2023-10-09T04:45:12.612202+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 33785/226494 objects degraded (14.917%); 725 KiB/s, 11 objects/s recovering 2023-10-09T04:45:14.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:14.227 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:14.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:14.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:14.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:14.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:14.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:14.619 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:14.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:14.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:14.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:14.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:14.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:14.993 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:15.015 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:15.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:15.152 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:15.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:15.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:15.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:15.422 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[77063]: audit 2023-10-09T04:45:14.289806+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:15.422 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[77063]: audit 2023-10-09T04:45:14.296789+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:15.422 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[77063]: cluster 2023-10-09T04:45:14.612898+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 33147/220743 objects degraded (15.016%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:15.423 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[77063]: cluster 2023-10-09T04:45:14.946967+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 33785/226494 objects degraded (14.917%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:15.423 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[79733]: audit 2023-10-09T04:45:14.289806+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:15.423 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[79733]: audit 2023-10-09T04:45:14.296789+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:15.423 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[79733]: cluster 2023-10-09T04:45:14.612898+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 33147/220743 objects degraded (15.016%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:15.423 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:15 smithi082 bash[79733]: cluster 2023-10-09T04:45:14.946967+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 33785/226494 objects degraded (14.917%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:15.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:15.646 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:15.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:15 smithi153 bash[59565]: audit 2023-10-09T04:45:14.289806+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:15.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:15 smithi153 bash[59565]: audit 2023-10-09T04:45:14.296789+0000 mon.a (mon.0) 1390 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:15.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:15 smithi153 bash[59565]: cluster 2023-10-09T04:45:14.612898+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 33147/220743 objects degraded (15.016%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:15.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:15 smithi153 bash[59565]: cluster 2023-10-09T04:45:14.946967+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 33785/226494 objects degraded (14.917%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:15.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:15.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:15.903 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:15.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:15.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:16.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:16.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:16.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:16.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:16.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:16.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:16.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:16.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:16.614 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:16.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:16.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:16.779 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:16.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:16.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:16.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:16.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:17.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:17.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:17.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:17.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:17.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:17.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:17.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:17.429 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:17.461 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:17.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:17.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:17.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:17.704 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:17 smithi153 bash[59565]: cluster 2023-10-09T04:45:16.613594+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 33147/217356 objects degraded (15.250%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:17.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:17.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:17.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:17.957 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:17.966 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:17 smithi082 bash[77063]: cluster 2023-10-09T04:45:16.613594+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 33147/217356 objects degraded (15.250%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:17.966 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:17 smithi082 bash[79733]: cluster 2023-10-09T04:45:16.613594+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 33147/217356 objects degraded (15.250%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:17.980 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:17.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:18.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:18.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:18.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:18.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:18.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:18.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:18.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:18.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:18.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:18.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:18.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:18.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:18.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:19.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:19.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:19.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:19.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:19.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:19.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:19.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:19.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:19.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:19.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:19.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:19.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:19.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:19.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:19.891 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:19.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:19.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:19 smithi082 bash[77063]: cluster 2023-10-09T04:45:18.615309+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 33147/216207 objects degraded (15.331%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:19.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:19] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-09T04:45:19.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:19 smithi082 bash[79733]: cluster 2023-10-09T04:45:18.615309+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 33147/216207 objects degraded (15.331%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:19.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:20.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:20.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:20.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:20.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:19 smithi153 bash[59565]: cluster 2023-10-09T04:45:18.615309+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 33147/216207 objects degraded (15.331%); 800 KiB/s, 12 objects/s recovering 2023-10-09T04:45:20.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:20.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:20.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:20.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:20.430 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:20.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:20.586 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:20.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:20.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:20.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:20.826 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:20 smithi082 bash[77063]: cluster 2023-10-09T04:45:19.948132+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 33147/216207 objects degraded (15.331%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:20.827 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:20 smithi082 bash[79733]: cluster 2023-10-09T04:45:19.948132+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 33147/216207 objects degraded (15.331%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:20.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:20.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:20.964 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:20 smithi153 bash[59565]: cluster 2023-10-09T04:45:19.948132+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 33147/216207 objects degraded (15.331%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:20.980 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:20.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:21.086 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:21.121 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:45:21.121 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:45:21.121 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:7933 8.1 8:84f58bad:::benchmark_data_smithi082_175677_object7932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:42.331489+0000", 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 158.77295693600001, 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.428906283, 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.122 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 7933 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331489+0000", 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331489+0000", 2023-10-09T04:45:21.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331487+0000", 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331530+0000", 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3197e-05 2023-10-09T04:45:21.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331542+0000", 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1345000000000001e-05 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331550+0000", 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9540000000000004e-06 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331605+0000", 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4846000000000003e-05 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331737+0000", 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013222 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.126 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.331852+0000", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011574 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.332370+0000", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00051772400000000001 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760256+0000", 2023-10-09T04:45:21.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42788557900000002 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760298+0000", 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2701999999999999e-05 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760395+0000", 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6910999999999994e-05 2023-10-09T04:45:21.128 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:7943 8.1 8:80c58f92:::benchmark_data_smithi082_175677_object7942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:42.337479+0000", 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 158.76696717600001, 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42319480999999998, 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.129 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 7943 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337479+0000", 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337479+0000", 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337478+0000", 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337507+0000", 2023-10-09T04:45:21.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8668000000000001e-05 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337516+0000", 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3330000000000003e-06 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337522+0000", 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9780000000000002e-06 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337592+0000", 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0133000000000001e-05 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337723+0000", 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013087 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.337860+0000", 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013659100000000001 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.339061+0000", 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001201735 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760540+0000", 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42147836500000002 2023-10-09T04:45:21.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760594+0000", 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4351000000000002e-05 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760674+0000", 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9562999999999997e-05 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.135 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:7948 8.1 8:845a6043:::benchmark_data_smithi082_175677_object7947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:42.338308+0000", 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 158.76613846699999, 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42277344500000003, 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.136 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 7948 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338308+0000", 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338308+0000", 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.137 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338307+0000", 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338332+0000", 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5181000000000002e-05 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338342+0000", 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0465000000000001e-05 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338350+0000", 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5569999999999998e-06 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338406+0000", 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6743999999999997e-05 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338530+0000", 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012371600000000001 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.338677+0000", 2023-10-09T04:45:21.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000146828 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.339294+0000", 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00061690400000000004 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760946+0000", 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42165248799999999 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.760995+0000", 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8455000000000003e-05 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761081+0000", 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6168999999999994e-05 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.142 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:7959 8.1 8:8faa33c4:::benchmark_data_smithi082_175677_object7958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:42.346634+0000", 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 158.75781215999999, 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.41462254300000001, 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.143 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 7959 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346634+0000", 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346634+0000", 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346633+0000", 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346661+0000", 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7909999999999999e-05 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346672+0000", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1194e-05 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346681+0000", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0790000000000007e-06 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346720+0000", 2023-10-09T04:45:21.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9285000000000001e-05 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346816+0000", 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5538000000000006e-05 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.346912+0000", 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5575999999999994e-05 2023-10-09T04:45:21.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.347390+0000", 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000478522 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761135+0000", 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.41374474999999999 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.148 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761190+0000", 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4780000000000001e-05 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761256+0000", 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6824999999999998e-05 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:7967 8.1 8:8566ebb7:::benchmark_data_smithi082_175677_object7966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:42.356290+0000", 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 158.74815580800001, 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.40513640499999998, 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 7967 2023-10-09T04:45:21.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356290+0000", 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356290+0000", 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356289+0000", 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356318+0000", 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8453999999999999e-05 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356327+0000", 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5540000000000003e-06 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356335+0000", 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1930000000000004e-06 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356373+0000", 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8130999999999998e-05 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356467+0000", 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4166999999999995e-05 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.356560+0000", 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3078999999999994e-05 2023-10-09T04:45:21.154 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.357044+0000", 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00048378499999999998 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761297+0000", 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.40425297199999999 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.155 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761345+0000", 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8446000000000003e-05 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:42.761427+0000", 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1482000000000004e-05 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.156 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:38570 8.1 8:8882f615:::benchmark_data_smithi082_175677_object38569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:57.684609+0000", 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 143.41983658699999, 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28369771999999999, 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38570 2023-10-09T04:45:21.157 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684609+0000", 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684609+0000", 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.158 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684608+0000", 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684635+0000", 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6962999999999999e-05 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.159 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684649+0000", 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3168e-05 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684657+0000", 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5469999999999997e-06 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684710+0000", 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2986999999999999e-05 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.160 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684811+0000", 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000100881 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.684924+0000", 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000113477 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.161 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.686353+0000", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0014286909999999999 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968170+0000", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28181666399999999 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968230+0000", 2023-10-09T04:45:21.162 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0152000000000003e-05 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968307+0000", 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7189999999999998e-05 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:38574 8.1 8:8141ab34:::benchmark_data_smithi082_175677_object38573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:45:21.163 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:42:57.688840+0000", 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 143.415606371, 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.27965857599999999, 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 38574 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.164 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688840+0000", 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688840+0000", 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.165 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688838+0000", 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688868+0000", 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9972999999999999e-05 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688880+0000", 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1187e-05 2023-10-09T04:45:21.166 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688886+0000", 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4389999999999997e-06 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.688937+0000", 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1286999999999999e-05 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.167 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.689047+0000", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000109788 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.689168+0000", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000120969 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.690385+0000", 2023-10-09T04:45:21.168 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0012169800000000001 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968363+0000", 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.27797779099999997 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968415+0000", 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2104000000000001e-05 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.169 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:42:57.968498+0000", 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3313999999999995e-05 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112215 8.3 8:c92889b1:::benchmark_data_smithi082_175677_object112214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.170 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.591182+0000", 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.513263963, 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22378925899999999, 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112215 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.171 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591182+0000", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591182+0000", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591182+0000", 2023-10-09T04:45:21.172 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591204+0000", 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2833e-05 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591214+0000", 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2669999999999995e-06 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.173 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.591220+0000", 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9549999999999997e-06 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.596717+0000", 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0054973950000000004 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.174 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.596827+0000", 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011033 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.596939+0000", 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000111878 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.175 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.605899+0000", 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0089596229999999999 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.606089+0000", 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019024299999999999 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814863+0000", 2023-10-09T04:45:21.176 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20877432700000001 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814921+0000", 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8134000000000003e-05 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814971+0000", 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9870000000000002e-05 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.177 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112216 8.a 8:5ccdc89b:::benchmark_data_smithi082_175677_object112215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.593963+0000", 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.510483558000004, 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.215788179, 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.178 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112216 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.593963+0000", 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.179 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.593963+0000", 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.593961+0000", 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.593986+0000", 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4400999999999999e-05 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.180 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.593999+0000", 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3397999999999999e-05 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.594007+0000", 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1650000000000006e-06 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.181 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.594050+0000", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2846000000000003e-05 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.594151+0000", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010106 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.594291+0000", 2023-10-09T04:45:21.182 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013951499999999999 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.601439+0000", 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0071487440000000003 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809605+0000", 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20816522800000001 2023-10-09T04:45:21.183 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809657+0000", 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2386999999999998e-05 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809751+0000", 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3739000000000005e-05 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.184 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112219 8.1 8:84a6f9bc:::benchmark_data_smithi082_175677_object112218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.598022+0000", 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.506424363999997, 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.217616479, 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.185 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112219 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598022+0000", 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598022+0000", 2023-10-09T04:45:21.186 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598020+0000", 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598046+0000", 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5859000000000001e-05 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.187 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598059+0000", 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3606e-05 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598068+0000", 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5420000000000005e-06 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.188 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598118+0000", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9911999999999999e-05 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598264+0000", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014610299999999999 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598402+0000", 2023-10-09T04:45:21.189 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000137806 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.605118+0000", 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0067162819999999996 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815482+0000", 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21036376400000001 2023-10-09T04:45:21.190 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815537+0000", 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5090999999999998e-05 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815638+0000", 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000101235 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.191 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112220 8.f 8:ff77fb8c:::benchmark_data_smithi082_175677_object112219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.598087+0000", 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.506359508000003, 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21161740100000001, 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.192 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112220 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598087+0000", 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598087+0000", 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.193 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598086+0000", 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598112+0000", 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6393e-05 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.194 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598120+0000", 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4070000000000004e-06 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598125+0000", 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3279999999999996e-06 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.195 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598181+0000", 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5872000000000003e-05 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598310+0000", 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012868299999999999 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3", 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.598531+0000", 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00022169099999999999 2023-10-09T04:45:21.196 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809557+0000", 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21102536 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809565+0000", 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5799999999999992e-06 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.197 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809611+0000", 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6115000000000001e-05 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809704+0000", 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2602999999999994e-05 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.198 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112223 8.3 8:c4a0b627:::benchmark_data_smithi082_175677_object112222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.600828+0000", 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.503618161000006, 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21509664000000001, 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112223 2023-10-09T04:45:21.199 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600828+0000", 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600828+0000", 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.200 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600826+0000", 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600852+0000", 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5783e-05 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.201 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600864+0000", 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2133000000000001e-05 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600872+0000", 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8820000000000001e-06 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.600911+0000", 2023-10-09T04:45:21.202 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9422999999999998e-05 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.601062+0000", 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015033700000000001 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.601224+0000", 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000162397 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.203 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815004+0000", 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21378045300000001 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815047+0000", 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2904999999999997e-05 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.204 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815262+0000", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021437 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815880+0000", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00061884600000000004 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815925+0000", 2023-10-09T04:45:21.205 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4042999999999998e-05 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112224 8.3 8:c2b246ec:::benchmark_data_smithi082_175677_object112223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.602628+0000", 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.501818057999998, 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22014626000000001, 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.206 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112224 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602628+0000", 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.207 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602628+0000", 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602627+0000", 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.208 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602683+0000", 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5986999999999998e-05 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602695+0000", 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2333e-05 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602703+0000", 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1419999999999993e-06 2023-10-09T04:45:21.209 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809541+0000", 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20683823000000001 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809764+0000", 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00022260199999999999 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.210 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809952+0000", 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018768699999999999 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.822643+0000", 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012691345 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.211 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.822651+0000", 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.994e-06 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.822679+0000", 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8056999999999999e-05 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.822730+0000", 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.109e-05 2023-10-09T04:45:21.212 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.822774+0000", 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4121e-05 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112225 8.7 8:e0450f84:::benchmark_data_smithi082_175677_object112224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.602710+0000", 2023-10-09T04:45:21.213 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.501736235999999, 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21240930399999999, 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112225 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.214 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602710+0000", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602710+0000", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602709+0000", 2023-10-09T04:45:21.215 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602731+0000", 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1591999999999999e-05 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602737+0000", 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4389999999999997e-06 2023-10-09T04:45:21.216 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602743+0000", 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7429999999999999e-06 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602825+0000", 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1476000000000004e-05 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.217 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602952+0000", 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000127797 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603176+0000", 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000223904 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809716+0000", 2023-10-09T04:45:21.218 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20654024000000001 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814863+0000", 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.005146103 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815028+0000", 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016522999999999999 2023-10-09T04:45:21.219 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815068+0000", 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0553999999999998e-05 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815119+0000", 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.220 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:112218 8.0 e229/218 8:0140851c:::benchmark_data_smithi082_175677_object112217:head v 229'6993, mlcod=229'6991)", 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.602835+0000", 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.501611179999998, 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.206660978, 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.221 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602835+0000", 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602835+0000", 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602835+0000", 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967296 2023-10-09T04:45:21.222 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602880+0000", 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5439000000000003e-05 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602889+0000", 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0070000000000003e-06 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.223 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602894+0000", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0030000000000002e-06 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602940+0000", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5637999999999999e-05 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602961+0000", 2023-10-09T04:45:21.224 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1607000000000002e-05 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809435+0000", 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.206474026 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809496+0000", 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0482000000000001e-05 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.225 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112226 8.7 8:e77142b4:::benchmark_data_smithi082_175677_object112225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.602950+0000", 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.501495926000004, 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22631501100000001, 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.226 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112226 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602950+0000", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602950+0000", 2023-10-09T04:45:21.227 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602949+0000", 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602977+0000", 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7453999999999998e-05 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.228 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602989+0000", 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2685e-05 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.602999+0000", 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.023e-05 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.229 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809747+0000", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.206747598 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809885+0000", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013772799999999999 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.810022+0000", 2023-10-09T04:45:21.230 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013711800000000001 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.824389+0000", 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.014366666 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.824664+0000", 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027578699999999999 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.231 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829173+0000", 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0045082869999999997 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829223+0000", 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0730999999999999e-05 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.232 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829265+0000", 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1792999999999998e-05 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:112227 8.7 8:e46f2736:::benchmark_data_smithi082_175677_object112226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.603150+0000", 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.501295631999994, 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22631044, 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.233 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 112227 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603150+0000", 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.234 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603150+0000", 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603150+0000", 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.235 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603176+0000", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6050999999999998e-05 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603186+0000", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.025e-05 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.603315+0000", 2023-10-09T04:45:21.236 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012904999999999999 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815167+0000", 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.21185173500000001 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815382+0000", 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021571900000000001 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.237 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.815585+0000", 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020246000000000001 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.824408+0000", 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0088230329999999992 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.238 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829277+0000", 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0048689919999999999 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829385+0000", 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000108082 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829424+0000", 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8685e-05 2023-10-09T04:45:21.239 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.829461+0000", 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7175000000000002e-05 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.35442.0:112225 8.7 e229/218 ondisk, result = 0)", 2023-10-09T04:45:21.240 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:03.607251+0000", 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 77.497195520999995, 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20761796199999999, 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.607251+0000", 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.241 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.607251+0000", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.607250+0000", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.607255+0000", 2023-10-09T04:45:21.242 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4650000000000002e-06 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809423+0000", 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.20216788399999999 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.809447+0000", 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4253e-05 2023-10-09T04:45:21.243 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814845+0000", 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053978969999999996 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814856+0000", 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.03e-05 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.244 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:03.814869+0000", 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2852e-05 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:153737 8.1 8:8e7da7df:::benchmark_data_smithi082_175677_object36723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:01.295109+0000", 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 19.809337316000001, 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.234273169, 2023-10-09T04:45:21.245 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 153737 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295109+0000", 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.246 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295109+0000", 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295107+0000", 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.247 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295114+0000", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.7959999999999998e-06 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295128+0000", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3984e-05 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295140+0000", 2023-10-09T04:45:21.248 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1839e-05 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.295976+0000", 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00083601300000000003 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296078+0000", 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000102539 2023-10-09T04:45:21.249 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296171+0000", 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2096999999999996e-05 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.303613+0000", 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0074421519999999996 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.250 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.251 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.303647+0000", 2023-10-09T04:45:21.251 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3806000000000001e-05 2023-10-09T04:45:21.251 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.251 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529145+0000", 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.225498431 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529275+0000", 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013029 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.252 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529382+0000", 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000106662 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:153741 8.1 8:88722814:::benchmark_data_smithi082_175677_object36727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:01.296703+0000", 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 19.807742705999999, 2023-10-09T04:45:21.253 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.23285873000000001, 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 153741 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:45:21.254 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296703+0000", 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296703+0000", 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296702+0000", 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:45:21.255 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296707+0000", 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9520000000000002e-06 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296718+0000", 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0756e-05 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.256 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.296727+0000", 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4470000000000006e-06 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.300006+0000", 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0032793509999999998 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:45:21.257 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.300176+0000", 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00017008600000000001 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.300326+0000", 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000150116 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.303641+0000", 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003314469 2023-10-09T04:45:21.258 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.303709+0000", 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8182999999999994e-05 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529433+0000", 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.22572396 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.259 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529473+0000", 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0098999999999997e-05 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:01.529562+0000", 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9190999999999994e-05 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:45:21.260 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:45:21.261 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:45:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:21.362 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:21.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:21.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:21.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:21.526 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:45:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:21.594 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:45:21.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:21.638 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:45:21.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:21.675 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:45:21.719 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:45:21.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:21.766 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:45:21.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:21.805 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:45:21.844 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:45:21.847 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.847 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.847 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.848 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.848 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.848 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.848 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.848 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:45:21.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:21.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:21.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:21.969 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:45:21.973 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:45:22.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:22.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:22.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:22.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[77063]: cluster 2023-10-09T04:45:20.617778+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 32517/208026 objects degraded (15.631%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[77063]: audit 2023-10-09T04:45:21.440954+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.82:0/151689258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[77063]: audit 2023-10-09T04:45:21.744290+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.82:0/3311461793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[77063]: audit 2023-10-09T04:45:21.746039+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[77063]: audit 2023-10-09T04:45:21.847696+0000 mon.a (mon.0) 1394 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[79733]: cluster 2023-10-09T04:45:20.617778+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 32517/208026 objects degraded (15.631%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[79733]: audit 2023-10-09T04:45:21.440954+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.82:0/151689258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[79733]: audit 2023-10-09T04:45:21.744290+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.82:0/3311461793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[79733]: audit 2023-10-09T04:45:21.746039+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:21 smithi082 bash[79733]: audit 2023-10-09T04:45:21.847696+0000 mon.a (mon.0) 1394 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:22.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:22.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:22.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:21 smithi153 bash[59565]: cluster 2023-10-09T04:45:20.617778+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 32517/208026 objects degraded (15.631%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:21 smithi153 bash[59565]: audit 2023-10-09T04:45:21.440954+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.82:0/151689258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:21 smithi153 bash[59565]: audit 2023-10-09T04:45:21.744290+0000 mon.b (mon.1) 25 : audit 1 from='client.? 172.21.15.82:0/3311461793' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:21 smithi153 bash[59565]: audit 2023-10-09T04:45:21.746039+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:45:22.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:21 smithi153 bash[59565]: audit 2023-10-09T04:45:21.847696+0000 mon.a (mon.0) 1394 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:22.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:22.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:22.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:22.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:22.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:22.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:22.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:22.914 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:22.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:23.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:23.043 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:23.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: cluster 2023-10-09T04:45:21.952369+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: cluster 2023-10-09T04:45:21.956066+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: cluster 2023-10-09T04:45:21.960492+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: audit 2023-10-09T04:45:21.960802+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: audit 2023-10-09T04:45:21.961054+0000 mon.a (mon.0) 1397 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:23.155 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[77063]: cluster 2023-10-09T04:45:21.961628+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: cluster 2023-10-09T04:45:21.952369+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: cluster 2023-10-09T04:45:21.956066+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: cluster 2023-10-09T04:45:21.960492+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: audit 2023-10-09T04:45:21.960802+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: audit 2023-10-09T04:45:21.961054+0000 mon.a (mon.0) 1397 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:23.156 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:22 smithi082 bash[79733]: cluster 2023-10-09T04:45:21.961628+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:45:23.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:23.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:23.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:23.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: cluster 2023-10-09T04:45:21.952369+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: cluster 2023-10-09T04:45:21.956066+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: cluster 2023-10-09T04:45:21.960492+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T04:45:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: audit 2023-10-09T04:45:21.960802+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:45:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: audit 2023-10-09T04:45:21.961054+0000 mon.a (mon.0) 1397 : audit 1 from='client.? 172.21.15.82:0/1379491732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:22 smithi153 bash[59565]: cluster 2023-10-09T04:45:21.961628+0000 mon.a (mon.0) 1398 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:45:23.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:23.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:23.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:23.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:23.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:23.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:23.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:23.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:23.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:23.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:23.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:24.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:45:24.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:24.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:24.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:24.246 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:23 smithi153 bash[59565]: cluster 2023-10-09T04:45:22.618178+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 32517/208026 objects degraded (15.631%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:45:24.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:24.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:24.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:23 smithi082 bash[77063]: cluster 2023-10-09T04:45:22.618178+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 32517/208026 objects degraded (15.631%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:45:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:23 smithi082 bash[79733]: cluster 2023-10-09T04:45:22.618178+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 32517/208026 objects degraded (15.631%); 991 KiB/s, 15 objects/s recovering 2023-10-09T04:45:24.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:24.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:24.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:24.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:24.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:24.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:24.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:24.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:24.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:25.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:25.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:25.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:25.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:25.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:24 smithi082 bash[77063]: cluster 2023-10-09T04:45:24.949479+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 32517/208026 objects degraded (15.631%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:25.187 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:24 smithi082 bash[79733]: cluster 2023-10-09T04:45:24.949479+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 32517/208026 objects degraded (15.631%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:25.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:25.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:25.334 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:24 smithi153 bash[59565]: cluster 2023-10-09T04:45:24.949479+0000 mon.a (mon.0) 1399 : cluster 3 Health check update: Degraded data redundancy: 32517/208026 objects degraded (15.631%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:25.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:25.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:25.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:25.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:25.644 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:25.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:25.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:25.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:25.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:25.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:26.000 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:25 smithi082 bash[77063]: cluster 2023-10-09T04:45:24.618830+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 31888/202380 objects degraded (15.756%); 998 KiB/s, 15 objects/s recovering 2023-10-09T04:45:26.000 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:25 smithi082 bash[79733]: cluster 2023-10-09T04:45:24.618830+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 31888/202380 objects degraded (15.756%); 998 KiB/s, 15 objects/s recovering 2023-10-09T04:45:26.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:26.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:26.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:26.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:25 smithi153 bash[59565]: cluster 2023-10-09T04:45:24.618830+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 31888/202380 objects degraded (15.756%); 998 KiB/s, 15 objects/s recovering 2023-10-09T04:45:26.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:26.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:26.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:26.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:45:26.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:26.523 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:26.627 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:26.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:26.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:26.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:26.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:26.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:26.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:27.044 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:27.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:27.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:27.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:27.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:27.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:27.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:27.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:27.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:27.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:27.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:27.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:27.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:27.759 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:27.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:27.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:27.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:27.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:27.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:28.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:28.012 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:28 smithi082 bash[77063]: cluster 2023-10-09T04:45:26.623649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 31888/198969 objects degraded (16.027%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:28.012 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:28 smithi082 bash[79733]: cluster 2023-10-09T04:45:26.623649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 31888/198969 objects degraded (16.027%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:28.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:28.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:28 smithi153 bash[59565]: cluster 2023-10-09T04:45:26.623649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 31888/198969 objects degraded (16.027%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:28.181 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:28.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:28.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:28.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:45:28.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:28.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:28.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:45:28.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:28.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:28.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:28.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:28.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:28.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:28.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:28.893 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:28.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:29.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:29.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:29.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:29.155 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:29.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:29.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:29.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:29.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:29.532 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:29.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:29.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:29.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:29.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:29.802 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:29.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:29.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:29.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:29.968 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:29] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-09T04:45:29.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:29.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:29.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:30.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:30.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[77063]: cluster 2023-10-09T04:45:28.625656+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 31888/197739 objects degraded (16.126%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[77063]: audit 2023-10-09T04:45:29.285994+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[77063]: cluster 2023-10-09T04:45:29.950589+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 31888/197739 objects degraded (16.126%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[79733]: cluster 2023-10-09T04:45:28.625656+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 31888/197739 objects degraded (16.126%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[79733]: audit 2023-10-09T04:45:29.285994+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:30.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:30 smithi082 bash[79733]: cluster 2023-10-09T04:45:29.950589+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 31888/197739 objects degraded (16.126%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:30.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:30.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:30.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:30 smithi153 bash[59565]: cluster 2023-10-09T04:45:28.625656+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 31888/197739 objects degraded (16.126%); 997 KiB/s, 15 objects/s recovering 2023-10-09T04:45:30.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:30 smithi153 bash[59565]: audit 2023-10-09T04:45:29.285994+0000 mon.a (mon.0) 1400 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:30.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:30 smithi153 bash[59565]: cluster 2023-10-09T04:45:29.950589+0000 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 31888/197739 objects degraded (16.126%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:30.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:30.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:30.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:30.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:30.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:30.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:30.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:30.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:30.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:30.809 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:30.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:30.924 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:30.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:30.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:31.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:31.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:31.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:31.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:31.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:31.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:31.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:31.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:31.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:31.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:31.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:31.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:31.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:31.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:31.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:31.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:32.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.115 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:32.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:32.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:32.374 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:32 smithi153 bash[59565]: cluster 2023-10-09T04:45:30.626396+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 31225/189744 objects degraded (16.456%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T04:45:32.399 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:32 smithi082 bash[77063]: cluster 2023-10-09T04:45:30.626396+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 31225/189744 objects degraded (16.456%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T04:45:32.399 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:32 smithi082 bash[79733]: cluster 2023-10-09T04:45:30.626396+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 31225/189744 objects degraded (16.456%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T04:45:32.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:32.503 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:32.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:32.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:32.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:32.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:32.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:32.947 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:32.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:32.961 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:32.962 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:33.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:33.087 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:33.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:33.223 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:33.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:33.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:33.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:33.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:33.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:33.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:33.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:33.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:33.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:33.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:33.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:33.885 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:33.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:33.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:33.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:34.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:34.027 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:34 smithi082 bash[77063]: cluster 2023-10-09T04:45:32.626837+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 31225/189744 objects degraded (16.456%); 960 KiB/s, 14 objects/s recovering 2023-10-09T04:45:34.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:34.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:34 smithi153 bash[59565]: cluster 2023-10-09T04:45:32.626837+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 31225/189744 objects degraded (16.456%); 960 KiB/s, 14 objects/s recovering 2023-10-09T04:45:34.192 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:34.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:34.311 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:34 smithi082 bash[79733]: cluster 2023-10-09T04:45:32.626837+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 31225/189744 objects degraded (16.456%); 960 KiB/s, 14 objects/s recovering 2023-10-09T04:45:34.345 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:34.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:34.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:34.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:34.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:34.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:34.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:34.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:34.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:34.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:34.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:35.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:35.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:35.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:35.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:35.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:35.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:35.301 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:35 smithi082 bash[77063]: cluster 2023-10-09T04:45:34.951728+0000 mon.a (mon.0) 1402 : cluster 3 Health check update: Degraded data redundancy: 31225/189744 objects degraded (16.456%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:35.301 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:35 smithi082 bash[79733]: cluster 2023-10-09T04:45:34.951728+0000 mon.a (mon.0) 1402 : cluster 3 Health check update: Degraded data redundancy: 31225/189744 objects degraded (16.456%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:35.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:35.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:35.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:35 smithi153 bash[59565]: cluster 2023-10-09T04:45:34.951728+0000 mon.a (mon.0) 1402 : cluster 3 Health check update: Degraded data redundancy: 31225/189744 objects degraded (16.456%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:35.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:35.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:35.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:35.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:35.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:35.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:35.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:35.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:35.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:35.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:35.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:36.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:36.119 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:36.129 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:36 smithi153 bash[59565]: cluster 2023-10-09T04:45:34.627495+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 30563/184221 objects degraded (16.590%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:36.149 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:36.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:36.264 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:36 smithi082 bash[77063]: cluster 2023-10-09T04:45:34.627495+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 30563/184221 objects degraded (16.590%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:36.265 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:36 smithi082 bash[79733]: cluster 2023-10-09T04:45:34.627495+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 30563/184221 objects degraded (16.590%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:36.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:36.400 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:36.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:36.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:36.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:36.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:36.698 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:36.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:36.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:36.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:36.977 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:36.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:37.084 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:37.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:37.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:37.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:37.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:37.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:37.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:37.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:37.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:37.654 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:37.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:37.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:37.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:37.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:37.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:38.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:38.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:38.170 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:38 smithi153 bash[59565]: cluster 2023-10-09T04:45:36.628161+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 30563/180852 objects degraded (16.899%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:38.199 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:38.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:38.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:38.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:38 smithi082 bash[77063]: cluster 2023-10-09T04:45:36.628161+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 30563/180852 objects degraded (16.899%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:38.447 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:38 smithi082 bash[79733]: cluster 2023-10-09T04:45:36.628161+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 30563/180852 objects degraded (16.899%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:38.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:38.577 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.606 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:38.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:38.744 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:38.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:38.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:38.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:38.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:38.977 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:39.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:39.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:39.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:39.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:39.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:39.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:39.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:39.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:39.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:39.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:39.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:39.796 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:39.806 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:39] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-09T04:45:39.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:39.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:39.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:39.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:40.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:40.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:40.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:40.212 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:40 smithi153 bash[59565]: cluster 2023-10-09T04:45:38.629506+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 30563/179712 objects degraded (17.007%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:40.212 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:40 smithi153 bash[59565]: cluster 2023-10-09T04:45:39.952776+0000 mon.a (mon.0) 1403 : cluster 3 Health check update: Degraded data redundancy: 30563/179712 objects degraded (17.007%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:40.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:40 smithi082 bash[77063]: cluster 2023-10-09T04:45:38.629506+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 30563/179712 objects degraded (17.007%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:40.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:40 smithi082 bash[77063]: cluster 2023-10-09T04:45:39.952776+0000 mon.a (mon.0) 1403 : cluster 3 Health check update: Degraded data redundancy: 30563/179712 objects degraded (17.007%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:40 smithi082 bash[79733]: cluster 2023-10-09T04:45:38.629506+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 30563/179712 objects degraded (17.007%); 874 KiB/s, 13 objects/s recovering 2023-10-09T04:45:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:40 smithi082 bash[79733]: cluster 2023-10-09T04:45:39.952776+0000 mon.a (mon.0) 1403 : cluster 3 Health check update: Degraded data redundancy: 30563/179712 objects degraded (17.007%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:40.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:40.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:40.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:40.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:40.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:40.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:40.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.668 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:40.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:40.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:40.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:41.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:41.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:41.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:41.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:41.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:41.379 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:41.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:41.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:41.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:41.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:41.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:41.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:41.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:41.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:41.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:42.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:42.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:42.171 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:42 smithi082 bash[79733]: cluster 2023-10-09T04:45:40.630152+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 29899/171681 objects degraded (17.415%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:42.172 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:42 smithi082 bash[77063]: cluster 2023-10-09T04:45:40.630152+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 29899/171681 objects degraded (17.415%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:42.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:42.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:42.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:42.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:42.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:42 smithi153 bash[59565]: cluster 2023-10-09T04:45:40.630152+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1003: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 29899/171681 objects degraded (17.415%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:45:42.446 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:42.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:42.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:42.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:42.714 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:42.743 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:42.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:42.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:42.870 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:42.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:42.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:42.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:42.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:43.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:43.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:43.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:43.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:43.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:43.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:43.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:43.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:43.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:43.665 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:43.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:43.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:43.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:43.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:43.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:43.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:44.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:44.076 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:44 smithi082 bash[77063]: cluster 2023-10-09T04:45:42.630958+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 29899/171681 objects degraded (17.415%); 922 KiB/s, 14 objects/s recovering 2023-10-09T04:45:44.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:44 smithi082 bash[79733]: cluster 2023-10-09T04:45:42.630958+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 29899/171681 objects degraded (17.415%); 922 KiB/s, 14 objects/s recovering 2023-10-09T04:45:44.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:44.201 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:44.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:44.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:44.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:44.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:44 smithi153 bash[59565]: cluster 2023-10-09T04:45:42.630958+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1004: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 29899/171681 objects degraded (17.415%); 922 KiB/s, 14 objects/s recovering 2023-10-09T04:45:44.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:44.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:44.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:44.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:44.536 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:45:44.536 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:45:44.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:44.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:44.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:44.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:44.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:44.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:44.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:45.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:45.064 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:45 smithi082 bash[77063]: audit 2023-10-09T04:45:44.285953+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:45.064 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:45 smithi082 bash[77063]: cluster 2023-10-09T04:45:44.953739+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 29899/171681 objects degraded (17.415%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:45.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:45.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:45 smithi153 bash[59565]: audit 2023-10-09T04:45:44.285953+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:45 smithi153 bash[59565]: cluster 2023-10-09T04:45:44.953739+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 29899/171681 objects degraded (17.415%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:45.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:45.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:45.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:45 smithi082 bash[79733]: audit 2023-10-09T04:45:44.285953+0000 mon.a (mon.0) 1404 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:45.440 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:45 smithi082 bash[79733]: cluster 2023-10-09T04:45:44.953739+0000 mon.a (mon.0) 1405 : cluster 3 Health check update: Degraded data redundancy: 29899/171681 objects degraded (17.415%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:45.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:45.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:45.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:45.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:45.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:45.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:45.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:45.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:45.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:46.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:46.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:46.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:46.337 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:46 smithi082 bash[77063]: cluster 2023-10-09T04:45:44.631552+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 29289/166122 objects degraded (17.631%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:45:46.337 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:46 smithi082 bash[79733]: cluster 2023-10-09T04:45:44.631552+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 29289/166122 objects degraded (17.631%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:45:46.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:46.363 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:46 smithi153 bash[59565]: cluster 2023-10-09T04:45:44.631552+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 29289/166122 objects degraded (17.631%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:45:46.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:46.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:46.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:46.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:46.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:46.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:46.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:46.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:46.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:46.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:46.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:47.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:47.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:47.086 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:45:47.174 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:47.181 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:47 smithi082 bash[77063]: audit 2023-10-09T04:45:47.055029+0000 mon.a (mon.0) 1406 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:45:47.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:47 smithi082 bash[79733]: audit 2023-10-09T04:45:47.055029+0000 mon.a (mon.0) 1406 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:45:47.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:47.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:47.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:47 smithi153 bash[59565]: audit 2023-10-09T04:45:47.055029+0000 mon.a (mon.0) 1406 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:45:47.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:47.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:47.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:47.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:47.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:47.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:47.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:47.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:47.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:47.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:47.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:48.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:48.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:48.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:45:48.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: cluster 2023-10-09T04:45:46.632229+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1006: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 29289/162675 objects degraded (18.005%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: cluster 2023-10-09T04:45:47.077479+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: audit 2023-10-09T04:45:47.078896+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: cluster 2023-10-09T04:45:47.079344+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: cluster 2023-10-09T04:45:47.085634+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:48 smithi153 bash[59565]: cluster 2023-10-09T04:45:47.086481+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:48.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:48.322 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: cluster 2023-10-09T04:45:46.632229+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1006: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 29289/162675 objects degraded (18.005%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: cluster 2023-10-09T04:45:47.077479+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: audit 2023-10-09T04:45:47.078896+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: cluster 2023-10-09T04:45:47.079344+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: cluster 2023-10-09T04:45:47.085634+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[77063]: cluster 2023-10-09T04:45:47.086481+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:48.329 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: cluster 2023-10-09T04:45:46.632229+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1006: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 29289/162675 objects degraded (18.005%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:45:48.329 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: cluster 2023-10-09T04:45:47.077479+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.329 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: audit 2023-10-09T04:45:47.078896+0000 mon.a (mon.0) 1407 : audit 1 from='client.? 172.21.15.82:0/2350969646' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:45:48.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: cluster 2023-10-09T04:45:47.079344+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:45:48.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: cluster 2023-10-09T04:45:47.085634+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:48 smithi082 bash[79733]: cluster 2023-10-09T04:45:47.086481+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T04:45:48.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:48.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:48.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:48.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:48.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:48.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:48.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:49.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:49.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:49.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:49 smithi153 bash[59565]: cluster 2023-10-09T04:45:48.632780+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 29289/161535 objects degraded (18.132%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:49.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:49 smithi153 bash[59565]: audit 2023-10-09T04:45:48.680027+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:49.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:49.146 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:49.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:49 smithi082 bash[79733]: cluster 2023-10-09T04:45:48.632780+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 29289/161535 objects degraded (18.132%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:49 smithi082 bash[79733]: audit 2023-10-09T04:45:48.680027+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:49.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:49 smithi082 bash[77063]: cluster 2023-10-09T04:45:48.632780+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1008: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 29289/161535 objects degraded (18.132%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:45:49.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:49 smithi082 bash[77063]: audit 2023-10-09T04:45:48.680027+0000 mon.a (mon.0) 1409 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:45:49.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:49.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:49.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:49.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:49.520 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:49.552 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:49.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:49.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:49.680 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:49.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:49.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:49.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:49.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:49.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:49] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-10-09T04:45:50.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:50.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:50.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:50.100 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:50 smithi153 bash[59565]: cluster 2023-10-09T04:45:49.954804+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 29289/161535 objects degraded (18.132%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:50.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:50.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:50.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:50.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.362 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:50.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:50 smithi082 bash[77063]: cluster 2023-10-09T04:45:49.954804+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 29289/161535 objects degraded (18.132%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:50 smithi082 bash[79733]: cluster 2023-10-09T04:45:49.954804+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 29289/161535 objects degraded (18.132%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:50.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:50.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:50.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:50.769 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:50.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:50.872 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:50.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:51.003 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:51.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:51.111 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:51 smithi082 bash[77063]: cluster 2023-10-09T04:45:50.633533+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:51.111 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:51 smithi082 bash[79733]: cluster 2023-10-09T04:45:50.633533+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:51.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:51.164 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:51 smithi153 bash[59565]: cluster 2023-10-09T04:45:50.633533+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1009: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:51.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:51.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:51.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:51.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:51.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:51.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:45:51.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.632 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:51.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:51.733 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:51.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:51.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:51.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:51.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:51.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:52.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:52.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:52.192 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:52.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:52.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:52.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:52.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:52.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:52.796 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:45:52.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:52.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:52.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:52.869 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:45:52.925 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:45:52.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:52.971 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:45:52.984 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:52.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:53.022 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:45:53.065 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:45:53.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:53.102 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:45:53.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:53.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:53.136 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:45:53.138 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.138 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.139 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:45:53.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:53.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:53.340 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:53.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:53.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:53.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:53.491 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:53.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:53.621 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:53.702 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:45:53.708 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[77063]: cluster 2023-10-09T04:45:52.634014+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:53.708 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[77063]: audit 2023-10-09T04:45:52.734328+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.82:0/2681018327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:53.709 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[77063]: audit 2023-10-09T04:45:53.082321+0000 mon.a (mon.0) 1411 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:53.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[79733]: cluster 2023-10-09T04:45:52.634014+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:53.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[79733]: audit 2023-10-09T04:45:52.734328+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.82:0/2681018327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:53.709 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:53 smithi082 bash[79733]: audit 2023-10-09T04:45:53.082321+0000 mon.a (mon.0) 1411 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:53.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:53.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:53.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:53.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:53.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:53 smithi153 bash[59565]: cluster 2023-10-09T04:45:52.634014+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1010: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 28671/153738 objects degraded (18.649%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:45:53.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:53 smithi153 bash[59565]: audit 2023-10-09T04:45:52.734328+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.82:0/2681018327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:53.867 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:53 smithi153 bash[59565]: audit 2023-10-09T04:45:53.082321+0000 mon.a (mon.0) 1411 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:53.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:53.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:54.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:54.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:54.161 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:54.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:54.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:54.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:54.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:54.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:54.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:54.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:54.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:54.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:54.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:54.856 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.689823+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.696531+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: audit 2023-10-09T04:45:53.696968+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.697099+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.701969+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.702916+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.708580+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.863 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[77063]: cluster 2023-10-09T04:45:53.710728+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.689823+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.696531+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: audit 2023-10-09T04:45:53.696968+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.697099+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.701969+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.702916+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.708580+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.864 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:54 smithi082 bash[79733]: cluster 2023-10-09T04:45:53.710728+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:54.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:54.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:55.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.689823+0000 mon.a (mon.0) 1412 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:55.076 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.696531+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:55.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: audit 2023-10-09T04:45:53.696968+0000 mon.a (mon.0) 1413 : audit 1 from='client.? 172.21.15.82:0/324246801' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:55.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.697099+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:45:55.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.701969+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:55.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.702916+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:55.077 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.708580+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:55.078 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:54 smithi153 bash[59565]: cluster 2023-10-09T04:45:53.710728+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T04:45:55.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:55.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.270 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:55.371 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:55.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:55.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:55.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:55.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:55.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:55.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[77063]: cluster 2023-10-09T04:45:54.634661+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 28015/147912 objects degraded (18.940%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:55.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[77063]: cluster 2023-10-09T04:45:54.956003+0000 mon.a (mon.0) 1415 : cluster 3 Health check update: Degraded data redundancy: 28015/147912 objects degraded (18.940%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:55.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[77063]: audit 2023-10-09T04:45:55.294436+0000 mon.a (mon.0) 1416 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.791 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[77063]: audit 2023-10-09T04:45:55.313136+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[79733]: cluster 2023-10-09T04:45:54.634661+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 28015/147912 objects degraded (18.940%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:55.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[79733]: cluster 2023-10-09T04:45:54.956003+0000 mon.a (mon.0) 1415 : cluster 3 Health check update: Degraded data redundancy: 28015/147912 objects degraded (18.940%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:55.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[79733]: audit 2023-10-09T04:45:55.294436+0000 mon.a (mon.0) 1416 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.792 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:55 smithi082 bash[79733]: audit 2023-10-09T04:45:55.313136+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.809 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:55.915 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:55 smithi153 bash[59565]: cluster 2023-10-09T04:45:54.634661+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1012: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 28015/147912 objects degraded (18.940%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:45:55.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:55 smithi153 bash[59565]: cluster 2023-10-09T04:45:54.956003+0000 mon.a (mon.0) 1415 : cluster 3 Health check update: Degraded data redundancy: 28015/147912 objects degraded (18.940%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:55 smithi153 bash[59565]: audit 2023-10-09T04:45:55.294436+0000 mon.a (mon.0) 1416 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:55 smithi153 bash[59565]: audit 2023-10-09T04:45:55.313136+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:55.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:55.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:56.048 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:56.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:56.182 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:56.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:56.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:56.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:56.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:56.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:56.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:56.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:56.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:56.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:56.864 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:56.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:56.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:56.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:57.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:57.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:57.197 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:57.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:57.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:57.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:57.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:57.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:57.509 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:57.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:45:57.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:57.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:57.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:57.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:57.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:57.912 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:57 smithi153 bash[59565]: cluster 2023-10-09T04:45:56.635413+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 28015/144636 objects degraded (19.369%); 1018 KiB/s, 15 objects/s recovering 2023-10-09T04:45:57.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:57.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:57.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:57 smithi082 bash[77063]: cluster 2023-10-09T04:45:56.635413+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 28015/144636 objects degraded (19.369%); 1018 KiB/s, 15 objects/s recovering 2023-10-09T04:45:57.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:57 smithi082 bash[79733]: cluster 2023-10-09T04:45:56.635413+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1013: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 28015/144636 objects degraded (19.369%); 1018 KiB/s, 15 objects/s recovering 2023-10-09T04:45:58.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:58.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:58.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:58.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:58.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:58.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:58.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:58.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:45:58.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:58.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:58.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:58.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:58.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:58.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:58.911 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:58.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:58.950 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:58.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:59.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:59.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:59.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:59.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:59.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:59.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:59.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:59.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:59.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:59.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:59.608 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:59.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:58.509776+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:58.523829+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: cluster 2023-10-09T04:45:58.637581+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 28015/143481 objects degraded (19.525%); 972 KiB/s, 15 objects/s recovering 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:59.147823+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:59.149172+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:59.162198+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[77063]: audit 2023-10-09T04:45:59.286333+0000 mon.a (mon.0) 1423 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.733 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:45:59] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:58.509776+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:58.523829+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: cluster 2023-10-09T04:45:58.637581+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 28015/143481 objects degraded (19.525%); 972 KiB/s, 15 objects/s recovering 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:59.147823+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:59.149172+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:59.162198+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:45:59 smithi082 bash[79733]: audit 2023-10-09T04:45:59.286333+0000 mon.a (mon.0) 1423 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:59.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:58.509776+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:58.523829+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: cluster 2023-10-09T04:45:58.637581+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1014: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 28015/143481 objects degraded (19.525%); 972 KiB/s, 15 objects/s recovering 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:59.147823+0000 mon.a (mon.0) 1420 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:59.149172+0000 mon.a (mon.0) 1421 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:59.858 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:59.162198+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:45:59.859 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:45:59 smithi153 bash[59565]: audit 2023-10-09T04:45:59.286333+0000 mon.a (mon.0) 1423 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:45:59.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:59.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:00.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:00.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:00.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:00.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:00.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:00.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:00.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.558 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:00.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:00.689 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:00 smithi082 bash[77063]: cluster 2023-10-09T04:45:59.957414+0000 mon.a (mon.0) 1424 : cluster 3 Health check update: Degraded data redundancy: 28015/143481 objects degraded (19.525%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:00 smithi082 bash[79733]: cluster 2023-10-09T04:45:59.957414+0000 mon.a (mon.0) 1424 : cluster 3 Health check update: Degraded data redundancy: 28015/143481 objects degraded (19.525%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:00.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:00.805 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:00 smithi153 bash[59565]: cluster 2023-10-09T04:45:59.957414+0000 mon.a (mon.0) 1424 : cluster 3 Health check update: Degraded data redundancy: 28015/143481 objects degraded (19.525%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:00.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:00.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:00.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:00.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:01.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:01.096 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:01.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:01.229 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:01.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:01.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:01.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:01.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:01.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:01.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:01.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:01.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:01 smithi153 bash[59565]: cluster 2023-10-09T04:46:00.638361+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:01.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:01.759 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:01 smithi082 bash[77063]: cluster 2023-10-09T04:46:00.638361+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:01.760 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:01 smithi082 bash[79733]: cluster 2023-10-09T04:46:00.638361+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1015: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:01.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:01.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:01.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:01.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:02.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:02.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:02.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:02.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:02.313 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:02.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:02.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:02.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:02.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:02.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:02.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:02.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:02.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:02.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:02.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:02.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:02.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:03.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:03.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:03.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:03.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:03.290 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:03.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.422 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:03.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:03.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:03.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:03.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:03 smithi082 bash[77063]: cluster 2023-10-09T04:46:02.639414+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:03.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:03.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:03 smithi153 bash[59565]: cluster 2023-10-09T04:46:02.639414+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:03.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:03.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:03.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:03.993 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:03 smithi082 bash[79733]: cluster 2023-10-09T04:46:02.639414+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1016: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 27414/136089 objects degraded (20.144%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:04.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:04.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:04.209 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:04.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:04.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:04.342 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:04.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:04.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:04.509 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:04.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:04.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:04.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:04.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:04.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:04.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:04.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:05.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:05.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:05.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:05.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:05.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:05.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:05.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:05.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:05.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:05.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:05.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:05.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:05.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:05.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:05.853 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:05.869 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:05.870 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:05.888 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:05.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:05.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:05 smithi153 bash[59565]: cluster 2023-10-09T04:46:04.640122+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 26767/130392 objects degraded (20.528%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:46:05.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:05 smithi153 bash[59565]: cluster 2023-10-09T04:46:04.958534+0000 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 26767/130392 objects degraded (20.528%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:05.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:05 smithi082 bash[77063]: cluster 2023-10-09T04:46:04.640122+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 26767/130392 objects degraded (20.528%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:46:05.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:05 smithi082 bash[77063]: cluster 2023-10-09T04:46:04.958534+0000 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 26767/130392 objects degraded (20.528%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:05 smithi082 bash[79733]: cluster 2023-10-09T04:46:04.640122+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1017: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 26767/130392 objects degraded (20.528%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:46:05.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:05 smithi082 bash[79733]: cluster 2023-10-09T04:46:04.958534+0000 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 26767/130392 objects degraded (20.528%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:05.992 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:06.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:06.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:06.153 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:06.221 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:46:06.281 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:46:06.282 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:46:06.282 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:46:06.282 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:46:06.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:06.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:06.421 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:06.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:06.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:06.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:06.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:06.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:06.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:06.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:06.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:06.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:06.974 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:06.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:07.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:07.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:07.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:07.206 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:07.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:07.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:07.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:07.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:07.481 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:07.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:07.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:07.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:07.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:07.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:07.758 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:07 smithi082 bash[77063]: cluster 2023-10-09T04:46:06.640780+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 26767/126975 objects degraded (21.081%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:07.758 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:07 smithi082 bash[79733]: cluster 2023-10-09T04:46:06.640780+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 26767/126975 objects degraded (21.081%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:07.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:07.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:07.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:07.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:07.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:07 smithi153 bash[59565]: cluster 2023-10-09T04:46:06.640780+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1018: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 26767/126975 objects degraded (21.081%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:08.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:08.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:08.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:08.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:08.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:08.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:08.334 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:08.441 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:08.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:08.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:08.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:08.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:08.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:08.863 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:08.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:08.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:09.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:09.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:09.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:09.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:09.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:09.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:09.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:09.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.600 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:09.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:09.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:09.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:09.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:09 smithi153 bash[59565]: cluster 2023-10-09T04:46:08.641568+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 26767/125904 objects degraded (21.260%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:09.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:09.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:09.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:10.019 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:09 smithi082 bash[77063]: cluster 2023-10-09T04:46:08.641568+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 26767/125904 objects degraded (21.260%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:10.020 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:09] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:46:10.020 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:09 smithi082 bash[79733]: cluster 2023-10-09T04:46:08.641568+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1019: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 26767/125904 objects degraded (21.260%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:10.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.127 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:10.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:10.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.395 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:10.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:10.527 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:10.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:10.653 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:10.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:10.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:10.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:10.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:10.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:10.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:10.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:10 smithi082 bash[77063]: cluster 2023-10-09T04:46:09.959598+0000 mon.a (mon.0) 1426 : cluster 3 Health check update: Degraded data redundancy: 26767/125904 objects degraded (21.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:10.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:10 smithi082 bash[79733]: cluster 2023-10-09T04:46:09.959598+0000 mon.a (mon.0) 1426 : cluster 3 Health check update: Degraded data redundancy: 26767/125904 objects degraded (21.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:11.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.031 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:10 smithi153 bash[59565]: cluster 2023-10-09T04:46:09.959598+0000 mon.a (mon.0) 1426 : cluster 3 Health check update: Degraded data redundancy: 26767/125904 objects degraded (21.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:11.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:11.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:11.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:11.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:46:11.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:11.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:11.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:11.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:11.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:11.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:11.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:11.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:11.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:11.989 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:46:12.004 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[77063]: cluster 2023-10-09T04:46:10.642196+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 26116/118197 objects degraded (22.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:12.004 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[77063]: audit 2023-10-09T04:46:11.780740+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.82:0/2025122514' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.004 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[77063]: audit 2023-10-09T04:46:11.781384+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[79733]: cluster 2023-10-09T04:46:10.642196+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 26116/118197 objects degraded (22.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:12.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[79733]: audit 2023-10-09T04:46:11.780740+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.82:0/2025122514' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.005 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:11 smithi082 bash[79733]: audit 2023-10-09T04:46:11.781384+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:12.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:12.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:11 smithi153 bash[59565]: cluster 2023-10-09T04:46:10.642196+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1020: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 26116/118197 objects degraded (22.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:12.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:11 smithi153 bash[59565]: audit 2023-10-09T04:46:11.780740+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.82:0/2025122514' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.182 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:11 smithi153 bash[59565]: audit 2023-10-09T04:46:11.781384+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:46:12.182 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:12.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:12.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:12.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:12.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:12.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:12.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:12.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:12.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:12.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:12.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:12.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:12.906 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:12.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:13.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:13.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:13.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:13.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[77063]: cluster 2023-10-09T04:46:11.970866+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[77063]: cluster 2023-10-09T04:46:11.970996+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[77063]: audit 2023-10-09T04:46:11.971522+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:46:13.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[77063]: cluster 2023-10-09T04:46:11.971605+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[77063]: cluster 2023-10-09T04:46:11.987516+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[79733]: cluster 2023-10-09T04:46:11.970866+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[79733]: cluster 2023-10-09T04:46:11.970996+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[79733]: audit 2023-10-09T04:46:11.971522+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[79733]: cluster 2023-10-09T04:46:11.971605+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:46:13.162 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:12 smithi082 bash[79733]: cluster 2023-10-09T04:46:11.987516+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:12 smithi153 bash[59565]: cluster 2023-10-09T04:46:11.970866+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:12 smithi153 bash[59565]: cluster 2023-10-09T04:46:11.970996+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:12 smithi153 bash[59565]: audit 2023-10-09T04:46:11.971522+0000 mon.a (mon.0) 1428 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:46:13.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:12 smithi153 bash[59565]: cluster 2023-10-09T04:46:11.971605+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:46:13.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:12 smithi153 bash[59565]: cluster 2023-10-09T04:46:11.987516+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:13.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:13.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:13.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:13.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:13.427 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:13.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:13.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:13.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:13.713 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:13.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:13.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:13.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:13.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:13.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:14.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:14.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[77063]: cluster 2023-10-09T04:46:11.995888+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:14.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[77063]: cluster 2023-10-09T04:46:12.642687+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 26116/118197 objects degraded (22.095%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:14.037 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[79733]: cluster 2023-10-09T04:46:11.995888+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:14.037 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[79733]: cluster 2023-10-09T04:46:12.642687+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 26116/118197 objects degraded (22.095%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:14.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:14.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:13 smithi153 bash[59565]: cluster 2023-10-09T04:46:11.995888+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T04:46:14.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:13 smithi153 bash[59565]: cluster 2023-10-09T04:46:12.642687+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1022: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 26116/118197 objects degraded (22.095%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:14.255 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:14.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:14.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:14.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:14.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:14.568 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:14.674 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.703 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:14.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:14.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:14.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:14.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:14.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:14.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:15.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:15.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:14 smithi153 bash[59565]: audit 2023-10-09T04:46:14.286290+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:15.093 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:14 smithi153 bash[59565]: cluster 2023-10-09T04:46:14.960648+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 26116/118197 objects degraded (22.095%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:15.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:15.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:15.251 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:15.354 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:15.393 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[77063]: audit 2023-10-09T04:46:14.286290+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[77063]: cluster 2023-10-09T04:46:14.960648+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 26116/118197 objects degraded (22.095%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[79733]: audit 2023-10-09T04:46:14.286290+0000 mon.a (mon.0) 1430 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:14 smithi082 bash[79733]: cluster 2023-10-09T04:46:14.960648+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 26116/118197 objects degraded (22.095%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:15.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:15.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:15.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:15.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:15.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:46:15.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:15.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:15.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:15.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:15.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:15.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:16.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:16.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:16.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:16.222 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:16.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:16.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:15 smithi082 bash[77063]: cluster 2023-10-09T04:46:14.643367+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 25453/112398 objects degraded (22.645%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:16.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:16 smithi082 bash[79733]: cluster 2023-10-09T04:46:14.643367+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 25453/112398 objects degraded (22.645%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:16.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:16.335 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:15 smithi153 bash[59565]: cluster 2023-10-09T04:46:14.643367+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1023: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 25453/112398 objects degraded (22.645%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:16.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:16.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:16.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:16.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:16.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:16.629 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:16.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:16.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:16.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:16.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:16.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:16.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:16.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.029 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:17.132 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:17.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:17.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:17.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:17.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:17.714 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:17.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.875 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:17.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:17.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:18.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:18.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:18.120 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:18 smithi082 bash[77063]: cluster 2023-10-09T04:46:16.644029+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25453/108936 objects degraded (23.365%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:18.121 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:18 smithi082 bash[79733]: cluster 2023-10-09T04:46:16.644029+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25453/108936 objects degraded (23.365%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:18.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:18.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:18.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:18.266 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:18 smithi153 bash[59565]: cluster 2023-10-09T04:46:16.644029+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1024: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25453/108936 objects degraded (23.365%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:18.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:18.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:18.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:18.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:18.645 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:18.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:18.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:18.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:18.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:18.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:18.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:19.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:19.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:19.121 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:46:19.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:19.169 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:46:19.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:19.213 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:19.229 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:46:19.275 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:46:19.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:19.318 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:46:19.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:19.352 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:46:19.397 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:46:19.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:19.449 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:46:19.451 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.451 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.451 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.451 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.451 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.452 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.452 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.452 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:46:19.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:19.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:19.605 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:19.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:19.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:19.838 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:19] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-10-09T04:46:19.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:19.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:19.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:19.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:19.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:20.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:20.022 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:46:20.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:20.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:20.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[77063]: cluster 2023-10-09T04:46:18.644489+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 25453/107739 objects degraded (23.625%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:20.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[77063]: audit 2023-10-09T04:46:19.080390+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.82:0/3384852481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:20.158 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[77063]: audit 2023-10-09T04:46:19.577379+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.82:0/2929467081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[77063]: audit 2023-10-09T04:46:19.578573+0000 mon.a (mon.0) 1432 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[77063]: cluster 2023-10-09T04:46:19.961809+0000 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 25453/107739 objects degraded (23.625%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[79733]: cluster 2023-10-09T04:46:18.644489+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 25453/107739 objects degraded (23.625%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[79733]: audit 2023-10-09T04:46:19.080390+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.82:0/3384852481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[79733]: audit 2023-10-09T04:46:19.577379+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.82:0/2929467081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[79733]: audit 2023-10-09T04:46:19.578573+0000 mon.a (mon.0) 1432 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.159 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:20 smithi082 bash[79733]: cluster 2023-10-09T04:46:19.961809+0000 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 25453/107739 objects degraded (23.625%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:20.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:20.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:20.396 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:20.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:20 smithi153 bash[59565]: cluster 2023-10-09T04:46:18.644489+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1025: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 25453/107739 objects degraded (23.625%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:20.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:20 smithi153 bash[59565]: audit 2023-10-09T04:46:19.080390+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.82:0/3384852481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:20.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:20 smithi153 bash[59565]: audit 2023-10-09T04:46:19.577379+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.82:0/2929467081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:20 smithi153 bash[59565]: audit 2023-10-09T04:46:19.578573+0000 mon.a (mon.0) 1432 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:20.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:20 smithi153 bash[59565]: cluster 2023-10-09T04:46:19.961809+0000 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 25453/107739 objects degraded (23.625%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:20.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:20.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:20.564 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:20.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:20.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:20.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:20.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:20.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:20.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:20.974 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:20.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:20.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:21.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:21.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.005849+0000 mon.a (mon.0) 1434 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:21.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.010563+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.014249+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: audit 2023-10-09T04:46:20.014469+0000 mon.a (mon.0) 1435 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:21.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.014804+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:46:21.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.020527+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.086 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:21 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.024412+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:21.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:21.214 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.005849+0000 mon.a (mon.0) 1434 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.010563+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.014249+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: audit 2023-10-09T04:46:20.014469+0000 mon.a (mon.0) 1435 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.014804+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.020527+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.024412+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.005849+0000 mon.a (mon.0) 1434 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.010563+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.215 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.014249+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: audit 2023-10-09T04:46:20.014469+0000 mon.a (mon.0) 1435 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:21.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.014804+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:46:21.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.020527+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:21 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.024412+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T04:46:21.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:21.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:21.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:21.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:21.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:21.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:21.644 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:21.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:21.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:21.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:21.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:21.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:22.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:22.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:22.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:22.173 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:22.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:22.288 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:22 smithi153 bash[59565]: cluster 2023-10-09T04:46:20.645246+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 24797/99768 objects degraded (24.855%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:22.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:22.360 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:22 smithi082 bash[77063]: cluster 2023-10-09T04:46:20.645246+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 24797/99768 objects degraded (24.855%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:22.360 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:22 smithi082 bash[79733]: cluster 2023-10-09T04:46:20.645246+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1027: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 24797/99768 objects degraded (24.855%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:22.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:22.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:22.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:22.586 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:22.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:22.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:22.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:22.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:22.858 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:22.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:22.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:22.993 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:22.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:23.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:23.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:23.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:23.265 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:23.369 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:23.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:23.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:23.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:23.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:23.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:23.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:23.784 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:23.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:23.825 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:23.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:23.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:23.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:24.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.069 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:24 smithi082 bash[77063]: cluster 2023-10-09T04:46:22.645880+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24797/99768 objects degraded (24.855%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:24.070 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:24 smithi082 bash[79733]: cluster 2023-10-09T04:46:22.645880+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24797/99768 objects degraded (24.855%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:24.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:24.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:24.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:24.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:24 smithi153 bash[59565]: cluster 2023-10-09T04:46:22.645880+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1028: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 24797/99768 objects degraded (24.855%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:24.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:24.478 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:24.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.610 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:24.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:24.743 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:24.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:24.881 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:24.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:24.987 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:25.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:25.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:25.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:25 smithi153 bash[59565]: cluster 2023-10-09T04:46:24.964562+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 24797/99768 objects degraded (24.855%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:25.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:25.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:25.299 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:25 smithi082 bash[77063]: cluster 2023-10-09T04:46:24.964562+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 24797/99768 objects degraded (24.855%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:25.300 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:25 smithi082 bash[79733]: cluster 2023-10-09T04:46:24.964562+0000 mon.a (mon.0) 1437 : cluster 3 Health check update: Degraded data redundancy: 24797/99768 objects degraded (24.855%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:25.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.424 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:25.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:46:25.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:25.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:46:25.554 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:25.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:25.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:25.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:25.949 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:25.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:25.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:26.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:26 smithi082 bash[77063]: cluster 2023-10-09T04:46:24.646553+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1029: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 24168/93861 objects degraded (25.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:26.104 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:26 smithi082 bash[79733]: cluster 2023-10-09T04:46:24.646553+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1029: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 24168/93861 objects degraded (25.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:26.127 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:26.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.270 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:26.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:26 smithi153 bash[59565]: cluster 2023-10-09T04:46:24.646553+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1029: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 24168/93861 objects degraded (25.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:26.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:26.549 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:26.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:26.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:26.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:26.917 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.958 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:26.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:27.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:27.108 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:27.211 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:27.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:27.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:27.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:27.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:27.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:27.481 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:27.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:27.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:27.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:27.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:27.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:27.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:27.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:27.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:28.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:28.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:28.198 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:28 smithi082 bash[77063]: cluster 2023-10-09T04:46:26.647256+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 24168/90351 objects degraded (26.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:28 smithi082 bash[79733]: cluster 2023-10-09T04:46:26.647256+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 24168/90351 objects degraded (26.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:28.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:28 smithi153 bash[59565]: cluster 2023-10-09T04:46:26.647256+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1030: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 24168/90351 objects degraded (26.749%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:28.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:28.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.462 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:28.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:28.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:28.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:28.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:28.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:28.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:28.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:28.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:29.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:29.090 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:29.090 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:29.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:29.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:29.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:29.340 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:29.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:29.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:29.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:29.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:29.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:29.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:29.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:29.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:29.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:29.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:29.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:29] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T04:46:30.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:30.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:30.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:30.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:30.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:30.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:30.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:30 smithi153 bash[59565]: cluster 2023-10-09T04:46:28.649332+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24168/89277 objects degraded (27.071%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:30.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:30 smithi153 bash[59565]: audit 2023-10-09T04:46:29.291852+0000 mon.a (mon.0) 1438 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:46:30.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:30 smithi153 bash[59565]: audit 2023-10-09T04:46:29.293412+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:30.314 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:30 smithi153 bash[59565]: cluster 2023-10-09T04:46:29.965662+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 24168/89277 objects degraded (27.071%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:30.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[77063]: cluster 2023-10-09T04:46:28.649332+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24168/89277 objects degraded (27.071%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[77063]: audit 2023-10-09T04:46:29.291852+0000 mon.a (mon.0) 1438 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[77063]: audit 2023-10-09T04:46:29.293412+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[77063]: cluster 2023-10-09T04:46:29.965662+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 24168/89277 objects degraded (27.071%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[79733]: cluster 2023-10-09T04:46:28.649332+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1031: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 24168/89277 objects degraded (27.071%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:30.347 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[79733]: audit 2023-10-09T04:46:29.291852+0000 mon.a (mon.0) 1438 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:46:30.348 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[79733]: audit 2023-10-09T04:46:29.293412+0000 mon.a (mon.0) 1439 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:30.348 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:30 smithi082 bash[79733]: cluster 2023-10-09T04:46:29.965662+0000 mon.a (mon.0) 1440 : cluster 3 Health check update: Degraded data redundancy: 24168/89277 objects degraded (27.071%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:30.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:30.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:30.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:30.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:30.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:30.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:30.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:30.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:30.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:30.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:30.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:31.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:31.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:31.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:31.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:31.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:31.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:31.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:31.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:31.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:46:31.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:31.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:31.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:31.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:31.976 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:31.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:31.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:32.049 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:32.054 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:32.062 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:32 smithi153 bash[59565]: cluster 2023-10-09T04:46:30.650173+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 23543/81417 objects degraded (28.917%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:32.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:32.091 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:32 smithi082 bash[77063]: cluster 2023-10-09T04:46:30.650173+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 23543/81417 objects degraded (28.917%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:32.092 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:32 smithi082 bash[79733]: cluster 2023-10-09T04:46:30.650173+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1032: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 23543/81417 objects degraded (28.917%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:46:32.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:32.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:32.250 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:32.354 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:32.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:32.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:32.547 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:32.653 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:32.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:32.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:32.781 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:32.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:32.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:32.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:32.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:33.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:33.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:33.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:33.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:33.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:33.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:33.384 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:33.488 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:33.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:33.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:33.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:33.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:33.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:33.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:33.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:34.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:34.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:34 smithi153 bash[59565]: cluster 2023-10-09T04:46:32.650761+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23543/81417 objects degraded (28.917%); 981 KiB/s, 15 objects/s recovering 2023-10-09T04:46:34.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:34.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:34.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:34.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:34.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:34 smithi082 bash[77063]: cluster 2023-10-09T04:46:32.650761+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23543/81417 objects degraded (28.917%); 981 KiB/s, 15 objects/s recovering 2023-10-09T04:46:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:34 smithi082 bash[79733]: cluster 2023-10-09T04:46:32.650761+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1033: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 23543/81417 objects degraded (28.917%); 981 KiB/s, 15 objects/s recovering 2023-10-09T04:46:34.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:34.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:34.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:34.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:34.759 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:34.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:34.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:34.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:35.131 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:35.139 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:35 smithi153 bash[59565]: cluster 2023-10-09T04:46:34.966589+0000 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 23543/81417 objects degraded (28.917%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:35.153 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:35.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:35.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.267 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:35 smithi082 bash[77063]: cluster 2023-10-09T04:46:34.966589+0000 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 23543/81417 objects degraded (28.917%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:35.268 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:35 smithi082 bash[79733]: cluster 2023-10-09T04:46:34.966589+0000 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 23543/81417 objects degraded (28.917%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:35.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:35.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:35.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:35.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:35.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:35.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:35.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:35.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:35.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:35.936 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:35.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:35.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:36.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:36.081 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:36 smithi082 bash[77063]: cluster 2023-10-09T04:46:34.651448+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 22865/75483 objects degraded (30.292%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:46:36.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:36.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:36.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:36.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:36.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:36 smithi153 bash[59565]: cluster 2023-10-09T04:46:34.651448+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 22865/75483 objects degraded (30.292%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:46:36.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:36.407 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:36.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:36.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:36 smithi082 bash[79733]: cluster 2023-10-09T04:46:34.651448+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1034: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 22865/75483 objects degraded (30.292%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:46:36.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:36.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:36.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:36.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:36.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:36.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:36.914 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:36.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:37.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:37.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:37.096 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:46:37.164 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:37.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:37 smithi153 bash[59565]: audit 2023-10-09T04:46:36.900218+0000 mon.a (mon.0) 1442 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:46:37.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:37.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:37.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:37.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:37.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:37.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:37 smithi082 bash[77063]: audit 2023-10-09T04:46:36.900218+0000 mon.a (mon.0) 1442 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:46:37.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:37 smithi082 bash[79733]: audit 2023-10-09T04:46:36.900218+0000 mon.a (mon.0) 1442 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:46:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:37.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:37.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:37.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:37.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:37.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:37.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:38.026 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:38.129 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:38.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:38.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: cluster 2023-10-09T04:46:36.652024+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 22865/71910 objects degraded (31.797%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:38.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: cluster 2023-10-09T04:46:37.084433+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: audit 2023-10-09T04:46:37.085990+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:46:38.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: cluster 2023-10-09T04:46:37.086187+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: cluster 2023-10-09T04:46:37.094641+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[77063]: cluster 2023-10-09T04:46:37.102861+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: cluster 2023-10-09T04:46:36.652024+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 22865/71910 objects degraded (31.797%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: cluster 2023-10-09T04:46:37.084433+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: audit 2023-10-09T04:46:37.085990+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: cluster 2023-10-09T04:46:37.086187+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: cluster 2023-10-09T04:46:37.094641+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:38 smithi082 bash[79733]: cluster 2023-10-09T04:46:37.102861+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.255 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: cluster 2023-10-09T04:46:36.652024+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1035: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 22865/71910 objects degraded (31.797%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: cluster 2023-10-09T04:46:37.084433+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: audit 2023-10-09T04:46:37.085990+0000 mon.a (mon.0) 1443 : audit 1 from='client.? 172.21.15.82:0/2307363611' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: cluster 2023-10-09T04:46:37.086187+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: cluster 2023-10-09T04:46:37.094641+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.272 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:38 smithi153 bash[59565]: cluster 2023-10-09T04:46:37.102861+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T04:46:38.295 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:38.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:38.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:38.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:38.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:38.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:38.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:38.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:38.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:38.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:38.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:38.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:38.966 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:39.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:39.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:39.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:39.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:39 smithi082 bash[77063]: cluster 2023-10-09T04:46:38.652575+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 22865/70728 objects degraded (32.328%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:39 smithi082 bash[79733]: cluster 2023-10-09T04:46:38.652575+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 22865/70728 objects degraded (32.328%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:39.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:39.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:39.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:39.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:39.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:39 smithi153 bash[59565]: cluster 2023-10-09T04:46:38.652575+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1037: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 22865/70728 objects degraded (32.328%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:39.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:39.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:39.652 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:39.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:39.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:39.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:39.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:39.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:39.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:39.956 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:39] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T04:46:40.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:40.068 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:40.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:40.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:40 smithi082 bash[77063]: cluster 2023-10-09T04:46:39.967737+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 22865/70728 objects degraded (32.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:40.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:40 smithi082 bash[79733]: cluster 2023-10-09T04:46:39.967737+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 22865/70728 objects degraded (32.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:40.295 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:40.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.326 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:40 smithi153 bash[59565]: cluster 2023-10-09T04:46:39.967737+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 22865/70728 objects degraded (32.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:40.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:40.427 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:40.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:40.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:40.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:40.580 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:40.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:40.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:40.719 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:40.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:40.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:40.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:40.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:40.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:40.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:41.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:41.223 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:41.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:41.299 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:46:41.300 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:41 smithi082 bash[77063]: cluster 2023-10-09T04:46:40.653390+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:41.300 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:41 smithi082 bash[79733]: cluster 2023-10-09T04:46:40.653390+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:41.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:41.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:41 smithi153 bash[59565]: cluster 2023-10-09T04:46:40.653390+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1038: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:41.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:41.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:41.632 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:41.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:41.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:41.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:41.894 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:41.894 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:41.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.947 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:41.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:42.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:42.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:42.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:42.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:42.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:42.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:42.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:42.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:42.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:42.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:42.735 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:42.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:42.863 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:46:42.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:42.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:42.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:43.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:43.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:43.146 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:43.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:43.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:43.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:43.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:43.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:43.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:43.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:43.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:43.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:43.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:43.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:43.704 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:43.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:43.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:43.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:43.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:43.952 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:43 smithi082 bash[77063]: cluster 2023-10-09T04:46:42.656456+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:43.952 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:43 smithi082 bash[79733]: cluster 2023-10-09T04:46:42.656456+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:43.968 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:43.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:44.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:44.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:44.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:43 smithi153 bash[59565]: cluster 2023-10-09T04:46:42.656456+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1039: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 22176/62607 objects degraded (35.421%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:44.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:44.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:44.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:44.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:44.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:44.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:44.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:44.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:44.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:44.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:44.704 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:46:44.759 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:46:44.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:44.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:44.807 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:46:44.852 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:46:44.898 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:46:44.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:44.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:44 smithi082 bash[79733]: audit 2023-10-09T04:46:44.287411+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:44.917 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:44 smithi082 bash[79733]: audit 2023-10-09T04:46:44.649904+0000 mon.a (mon.0) 1447 : audit 0 from='client.? 172.21.15.82:0/2571495952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:44.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:44 smithi082 bash[77063]: audit 2023-10-09T04:46:44.287411+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:44.917 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:44 smithi082 bash[77063]: audit 2023-10-09T04:46:44.649904+0000 mon.a (mon.0) 1447 : audit 0 from='client.? 172.21.15.82:0/2571495952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:44.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:44.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:44.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:44.952 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:46:44.997 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:46:45.031 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.037 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.038 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.038 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:46:45.053 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:45.060 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:44 smithi153 bash[59565]: audit 2023-10-09T04:46:44.287411+0000 mon.a (mon.0) 1446 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:45.061 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:44 smithi153 bash[59565]: audit 2023-10-09T04:46:44.649904+0000 mon.a (mon.0) 1447 : audit 0 from='client.? 172.21.15.82:0/2571495952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:45.083 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:45.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:45.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:45.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:45.356 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:45.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:45.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:45.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:45.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:45.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:45.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:45.866 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:45.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[77063]: cluster 2023-10-09T04:46:44.657110+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 21501/57210 objects degraded (37.583%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:45.874 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[77063]: cluster 2023-10-09T04:46:44.968833+0000 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 21501/57210 objects degraded (37.583%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[77063]: audit 2023-10-09T04:46:45.111218+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.82:0/1872960842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[77063]: audit 2023-10-09T04:46:45.111882+0000 mon.a (mon.0) 1449 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[79733]: cluster 2023-10-09T04:46:44.657110+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 21501/57210 objects degraded (37.583%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[79733]: cluster 2023-10-09T04:46:44.968833+0000 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 21501/57210 objects degraded (37.583%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[79733]: audit 2023-10-09T04:46:45.111218+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.82:0/1872960842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:45.875 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:45 smithi082 bash[79733]: audit 2023-10-09T04:46:45.111882+0000 mon.a (mon.0) 1449 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:45.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:45.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:45.997 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:46:46.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:46.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:46.151 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:45 smithi153 bash[59565]: cluster 2023-10-09T04:46:44.657110+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1040: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 21501/57210 objects degraded (37.583%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:46.151 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:45 smithi153 bash[59565]: cluster 2023-10-09T04:46:44.968833+0000 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 21501/57210 objects degraded (37.583%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:46.151 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:45 smithi153 bash[59565]: audit 2023-10-09T04:46:45.111218+0000 mon.b (mon.1) 26 : audit 1 from='client.? 172.21.15.82:0/1872960842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:46.152 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:45 smithi153 bash[59565]: audit 2023-10-09T04:46:45.111882+0000 mon.a (mon.0) 1449 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:46.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:46.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:46.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:46.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:46.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:46.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:46.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:46.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:46.798 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:46 smithi153 bash[59565]: cluster 2023-10-09T04:46:45.980982+0000 mon.a (mon.0) 1450 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:46 smithi153 bash[59565]: audit 2023-10-09T04:46:45.987520+0000 mon.a (mon.0) 1451 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:46 smithi153 bash[59565]: cluster 2023-10-09T04:46:45.987651+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:46:46.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:46 smithi153 bash[59565]: cluster 2023-10-09T04:46:45.992600+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:46.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:46.956 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[77063]: cluster 2023-10-09T04:46:45.980982+0000 mon.a (mon.0) 1450 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:46.956 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[77063]: audit 2023-10-09T04:46:45.987520+0000 mon.a (mon.0) 1451 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:46.956 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[77063]: cluster 2023-10-09T04:46:45.987651+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:46:46.956 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[77063]: cluster 2023-10-09T04:46:45.992600+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:46.957 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[79733]: cluster 2023-10-09T04:46:45.980982+0000 mon.a (mon.0) 1450 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:46.957 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[79733]: audit 2023-10-09T04:46:45.987520+0000 mon.a (mon.0) 1451 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:46.957 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[79733]: cluster 2023-10-09T04:46:45.987651+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:46:46.957 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:46 smithi082 bash[79733]: cluster 2023-10-09T04:46:45.992600+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:46.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:46.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:47.084 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:47.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:47.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:47.255 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:47.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:47.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:47.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:47.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:47.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:47.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:47.703 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:47.760 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:47 smithi082 bash[77063]: cluster 2023-10-09T04:46:45.983387+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:47.760 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:47 smithi082 bash[77063]: cluster 2023-10-09T04:46:46.657708+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 21501/53877 objects degraded (39.908%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:46:47.760 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:47 smithi082 bash[79733]: cluster 2023-10-09T04:46:45.983387+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:47.760 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:47 smithi082 bash[79733]: cluster 2023-10-09T04:46:46.657708+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 21501/53877 objects degraded (39.908%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:46:47.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:47.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:47.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:47.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:47.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:48.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:48.049 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:47 smithi153 bash[59565]: cluster 2023-10-09T04:46:45.983387+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T04:46:48.049 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:47 smithi153 bash[59565]: cluster 2023-10-09T04:46:46.657708+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 21501/53877 objects degraded (39.908%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:46:48.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:48.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:48.313 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:48.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:48.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:48.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:48.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:48.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:48.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:48.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:48.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:48.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:48.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:49.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:49.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:46:49.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:49.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.169 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:49.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:49.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:49.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:49.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:49.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:49.806 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:49.808 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:49 smithi153 bash[59565]: cluster 2023-10-09T04:46:48.658296+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 21501/52599 objects degraded (40.877%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:49.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:49.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:49.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:46:49.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.963 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:49.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:49 smithi082 bash[77063]: cluster 2023-10-09T04:46:48.658296+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 21501/52599 objects degraded (40.877%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:49] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-09T04:46:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:49 smithi082 bash[79733]: cluster 2023-10-09T04:46:48.658296+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1043: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 21501/52599 objects degraded (40.877%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:50.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:50.121 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:50.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:50.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:50.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:50.390 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:50.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:50.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:50.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:50.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:50.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:50.768 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:50 smithi082 bash[77063]: cluster 2023-10-09T04:46:49.970039+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 21501/52599 objects degraded (40.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.768 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:50 smithi082 bash[79733]: cluster 2023-10-09T04:46:49.970039+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 21501/52599 objects degraded (40.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:50.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:50.903 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:50 smithi153 bash[59565]: cluster 2023-10-09T04:46:49.970039+0000 mon.a (mon.0) 1453 : cluster 3 Health check update: Degraded data redundancy: 21501/52599 objects degraded (40.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.923 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:50.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:51.026 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:51.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:51.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:51.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:51.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:51.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:51.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:51.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:51.744 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:51.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:51.878 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:51.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:51.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:52.105 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:46:52.105 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:46:52.105 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:46:52.105 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:46:52.105 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:46:52.117 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:52.125 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:51 smithi082 bash[77063]: cluster 2023-10-09T04:46:50.663293+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:52.125 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:51 smithi082 bash[79733]: cluster 2023-10-09T04:46:50.663293+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:52.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:52.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:52.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:52.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:51 smithi153 bash[59565]: cluster 2023-10-09T04:46:50.663293+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1044: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:52.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:52.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:52.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:52.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:52.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:52.698 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:52.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:52.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.962 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:52.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:53.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:53.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:46:53.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:53.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:53.261 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:53.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:53.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:53.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:53.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:53.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:53.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:53.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:53.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:54.012 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:46:54.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[77063]: cluster 2023-10-09T04:46:52.663882+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[77063]: audit 2023-10-09T04:46:53.879498+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.82:0/1385131048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[77063]: audit 2023-10-09T04:46:53.880098+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[79733]: cluster 2023-10-09T04:46:52.663882+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[79733]: audit 2023-10-09T04:46:53.879498+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.82:0/1385131048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.052 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:54 smithi082 bash[79733]: audit 2023-10-09T04:46:53.880098+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:54.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:54 smithi153 bash[59565]: cluster 2023-10-09T04:46:52.663882+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 933 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 20831/44646 objects degraded (46.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:46:54.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:54 smithi153 bash[59565]: audit 2023-10-09T04:46:53.879498+0000 mon.c (mon.2) 107 : audit 1 from='client.? 172.21.15.82:0/1385131048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:54 smithi153 bash[59565]: audit 2023-10-09T04:46:53.880098+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:46:54.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:54.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:54.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:54.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:54.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:54.482 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:54.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:54.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:54.736 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:54.766 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:54.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:54.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:54.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:54.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:54.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:55.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:55.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:55.141 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[77063]: cluster 2023-10-09T04:46:53.998150+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[77063]: cluster 2023-10-09T04:46:54.000625+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[77063]: audit 2023-10-09T04:46:54.002032+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[77063]: cluster 2023-10-09T04:46:54.002157+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[77063]: cluster 2023-10-09T04:46:54.971364+0000 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 20831/44646 objects degraded (46.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[79733]: cluster 2023-10-09T04:46:53.998150+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[79733]: cluster 2023-10-09T04:46:54.000625+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[79733]: audit 2023-10-09T04:46:54.002032+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:46:55.142 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[79733]: cluster 2023-10-09T04:46:54.002157+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:46:55.143 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:55 smithi082 bash[79733]: cluster 2023-10-09T04:46:54.971364+0000 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 20831/44646 objects degraded (46.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:55.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:55.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:55.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:55.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:55.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:55 smithi153 bash[59565]: cluster 2023-10-09T04:46:53.998150+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:55 smithi153 bash[59565]: cluster 2023-10-09T04:46:54.000625+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:55.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:55 smithi153 bash[59565]: audit 2023-10-09T04:46:54.002032+0000 mon.a (mon.0) 1455 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:46:55.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:55 smithi153 bash[59565]: cluster 2023-10-09T04:46:54.002157+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:46:55.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:55 smithi153 bash[59565]: cluster 2023-10-09T04:46:54.971364+0000 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 20831/44646 objects degraded (46.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:55.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:55.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:55.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:46:55.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:55.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:55.667 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:55.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:55.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:55.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:55.892 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:55.893 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:55.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:55.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:55.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:56.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:56.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:56 smithi153 bash[59565]: cluster 2023-10-09T04:46:54.026871+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:56 smithi153 bash[59565]: cluster 2023-10-09T04:46:54.664464+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 871 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 20734/41685 objects degraded (49.740%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:56.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:56 smithi082 bash[77063]: cluster 2023-10-09T04:46:54.026871+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:56.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:56 smithi082 bash[77063]: cluster 2023-10-09T04:46:54.664464+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 871 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 20734/41685 objects degraded (49.740%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:56 smithi082 bash[79733]: cluster 2023-10-09T04:46:54.026871+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T04:46:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:56 smithi082 bash[79733]: cluster 2023-10-09T04:46:54.664464+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1047: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 871 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 20734/41685 objects degraded (49.740%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:46:56.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:56.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:56.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:56.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:56.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:56.518 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:56.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:56.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:56.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:56.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:56.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:56.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:56.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:56.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:56.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:57.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:57.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:57.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:57.190 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:57.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:57.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:57.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:57.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:57.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:57.571 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:57.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:57.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:57.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:57.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:57.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:57.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:57.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:57.977 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:58.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:58.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:58.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:58.134 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:46:58 smithi153 bash[59565]: cluster 2023-10-09T04:46:56.665255+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 20158/35409 objects degraded (56.929%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:58.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:58.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.279 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:46:58 smithi082 bash[77063]: cluster 2023-10-09T04:46:56.665255+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 20158/35409 objects degraded (56.929%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:58.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:46:58 smithi082 bash[79733]: cluster 2023-10-09T04:46:56.665255+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1048: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 740 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 20158/35409 objects degraded (56.929%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:46:58.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:58.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:58.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:58.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:58.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:58.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:58.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:58.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:58.813 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:58.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:58.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:58.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:58.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:59.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:59.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:59.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:59.236 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:59.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:59.388 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:59.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:59.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:59.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:59.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:59.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:59.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:59.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:59.926 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:59.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:59.963 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:46:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:46:59] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:46:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:59.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:00.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.108 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:00.214 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:00.223 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[77063]: cluster 2023-10-09T04:46:58.665829+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 718 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20158/34329 objects degraded (58.720%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:47:00.223 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[77063]: audit 2023-10-09T04:46:59.247214+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[77063]: audit 2023-10-09T04:46:59.297832+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[77063]: audit 2023-10-09T04:46:59.299343+0000 mon.a (mon.0) 1460 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[77063]: cluster 2023-10-09T04:46:59.973922+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 20158/34329 objects degraded (58.720%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[79733]: cluster 2023-10-09T04:46:58.665829+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 718 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20158/34329 objects degraded (58.720%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[79733]: audit 2023-10-09T04:46:59.247214+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:00.224 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[79733]: audit 2023-10-09T04:46:59.297832+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:00.225 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[79733]: audit 2023-10-09T04:46:59.299343+0000 mon.a (mon.0) 1460 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:00.225 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:00 smithi082 bash[79733]: cluster 2023-10-09T04:46:59.973922+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 20158/34329 objects degraded (58.720%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:00.348 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:00.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:00 smithi153 bash[59565]: cluster 2023-10-09T04:46:58.665829+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1049: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 718 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20158/34329 objects degraded (58.720%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:47:00.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:00 smithi153 bash[59565]: audit 2023-10-09T04:46:59.247214+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:00.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:00 smithi153 bash[59565]: audit 2023-10-09T04:46:59.297832+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:00.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:00 smithi153 bash[59565]: audit 2023-10-09T04:46:59.299343+0000 mon.a (mon.0) 1460 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:00.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:00 smithi153 bash[59565]: cluster 2023-10-09T04:46:59.973922+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 20158/34329 objects degraded (58.720%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:00.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:00.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:00.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:00.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:00.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:00.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:00.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:01.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:01.045 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:01.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:01.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:01.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:01.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:01.456 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:01.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:01.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:01.745 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:01.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:01.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:01.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:01.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:02.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:02.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.126 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:02.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:02 smithi153 bash[59565]: cluster 2023-10-09T04:47:00.666706+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:02.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:02 smithi082 bash[77063]: cluster 2023-10-09T04:47:00.666706+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:02.265 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:02 smithi082 bash[79733]: cluster 2023-10-09T04:47:00.666706+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1050: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:02.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:02.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:02.518 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:02.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:7921 8.2 8:41ba544b:::benchmark_data_smithi082_175677_object7920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:42.329732+0000", 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 260.22141516400001, 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42791109900000002, 2023-10-09T04:47:02.560 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 7921 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329732+0000", 2023-10-09T04:47:02.561 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329732+0000", 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329731+0000", 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6592000000000001e-05 2023-10-09T04:47:02.562 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329807+0000", 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1384e-05 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329819+0000", 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4640000000000003e-06 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.563 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329825+0000", 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015762099999999999 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.329983+0000", 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000135838 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.330119+0000", 2023-10-09T04:47:02.564 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016509599999999999 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.330284+0000", 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00091310399999999998 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.331197+0000", 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0029201850000000001 2023-10-09T04:47:02.565 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.334117+0000", 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42331324599999998 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.757430+0000", 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010492699999999999 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.566 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.757535+0000", 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000108144 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.757643+0000", 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42791109900000002 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.567 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:38566 8.2 8:425131eb:::benchmark_data_smithi082_175677_object38565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:57.683519+0000", 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 244.86762847200001, 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27803882200000002, 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.568 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 38566 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683519+0000", 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683519+0000", 2023-10-09T04:47:02.569 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683517+0000", 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013751 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683655+0000", 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7220000000000005e-06 2023-10-09T04:47:02.570 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683664+0000", 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7740000000000001e-06 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683669+0000", 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8384999999999998e-05 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.571 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683728+0000", 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000187259 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.683915+0000", 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000217889 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.572 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.684133+0000", 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018742559999999999 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.686007+0000", 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033545670000000001 2023-10-09T04:47:02.573 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.689362+0000", 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.272055716 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.961417+0000", 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4351999999999996e-05 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.574 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.961492+0000", 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5897999999999998e-05 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:57.961558+0000", 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27803882200000002 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.575 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:89932 8.b 8:d758b612:::benchmark_data_smithi082_175677_object89931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:43:39.279212+0000", 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 203.271935716, 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14362733499999999, 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.576 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 89932 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279212+0000", 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279212+0000", 2023-10-09T04:47:02.577 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279210+0000", 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7658999999999997e-05 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279277+0000", 2023-10-09T04:47:02.578 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.37e-05 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279301+0000", 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0757999999999999e-05 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279312+0000", 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7249e-05 2023-10-09T04:47:02.579 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279369+0000", 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000150433 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279519+0000", 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018945499999999999 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.580 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.279709+0000", 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026814830000000001 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.282390+0000", 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0017999750000000001 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.581 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.284190+0000", 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13856896499999999 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.422759+0000", 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8203e-05 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.582 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.422798+0000", 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1452000000000002e-05 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.422839+0000", 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14362733499999999 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.583 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112237 8.b 8:d0532826:::benchmark_data_smithi082_175677_object112236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:03.617101+0000", 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 178.93404640099999, 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21876135299999999, 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112237 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.584 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617101+0000", 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617101+0000", 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.585 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617099+0000", 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8761000000000003e-05 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617148+0000", 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5386000000000001e-05 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.586 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617164+0000", 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8499999999999999e-06 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617169+0000", 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4570000000000001e-05 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617224+0000", 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011091600000000001 2023-10-09T04:47:02.587 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617335+0000", 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013194199999999999 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.617467+0000", 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034384900000000002 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.588 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.620905+0000", 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2782999999999999e-05 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.620918+0000", 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21479854600000001 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.589 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.835717+0000", 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6360000000000007e-05 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.835803+0000", 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9296999999999998e-05 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.835862+0000", 2023-10-09T04:47:02.590 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21876135299999999 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112239 8.b 8:dc9428f6:::benchmark_data_smithi082_175677_object112238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:03.621440+0000", 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 178.92970695899999, 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.214598127, 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.591 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112239 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621440+0000", 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.592 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621440+0000", 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621439+0000", 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5758999999999999e-05 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.593 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621495+0000", 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2962000000000001e-05 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621508+0000", 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2469999999999997e-06 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621514+0000", 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4046999999999999e-05 2023-10-09T04:47:02.594 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621568+0000", 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8530999999999996e-05 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621667+0000", 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014568 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.595 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.621812+0000", 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0030511689999999998 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.624863+0000", 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015002700000000001 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.596 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.625013+0000", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21090356800000001 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.835917+0000", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3483000000000002e-05 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.835980+0000", 2023-10-09T04:47:02.597 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8053999999999997e-05 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.836039+0000", 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.214598127 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.598 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112886 8.b 8:d693c31d:::benchmark_data_smithi082_175677_object112885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.546444+0000", 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 178.00470314200001, 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16251828900000001, 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112886 2023-10-09T04:47:02.599 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546444+0000", 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546444+0000", 2023-10-09T04:47:02.600 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546443+0000", 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0328999999999998e-05 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546504+0000", 2023-10-09T04:47:02.601 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2216e-05 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546516+0000", 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2340000000000002e-06 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.546521+0000", 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011874631 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.602 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558396+0000", 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018976099999999999 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558586+0000", 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018685199999999999 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.603 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558772+0000", 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080493649999999993 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.566822+0000", 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5865000000000001e-05 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.604 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.566878+0000", 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14179613399999999 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.708674+0000", 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017740900000000001 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.708851+0000", 2023-10-09T04:47:02.605 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000111383 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.708963+0000", 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16251828900000001 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.606 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112890 8.b 8:d7459af1:::benchmark_data_smithi082_175677_object112889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.553610+0000", 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.99753726500001, 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15558848, 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.607 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112890 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553610+0000", 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.608 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553610+0000", 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553608+0000", 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017426200000000001 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.609 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553782+0000", 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8668000000000002e-05 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553801+0000", 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.610 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.553809+0000", 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0086024680000000003 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.562411+0000", 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011404999999999999 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.562525+0000", 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014805999999999999 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.611 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.562673+0000", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0042587229999999998 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.566932+0000", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080286530000000002 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.574960+0000", 2023-10-09T04:47:02.612 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.134099156 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.709060+0000", 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9487000000000007e-05 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.709139+0000", 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9447999999999997e-05 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.613 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.709199+0000", 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15558848 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112896 8.b 8:d3195c94:::benchmark_data_smithi082_175677_object112895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.558189+0000", 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.99295882800001, 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.160230978, 2023-10-09T04:47:02.614 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112896 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.615 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558189+0000", 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558189+0000", 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558188+0000", 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2087000000000001e-05 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.616 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558280+0000", 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8539999999999992e-06 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558290+0000", 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.306e-06 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.617 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558297+0000", 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0086657820000000003 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.566963+0000", 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000149009 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.618 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.567112+0000", 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000179116 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.567291+0000", 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0077190519999999997 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.575010+0000", 2023-10-09T04:47:02.619 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.742e-06 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.575017+0000", 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14323574 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.718252+0000", 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1197000000000001e-05 2023-10-09T04:47:02.620 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.718343+0000", 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6039999999999997e-05 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.718420+0000", 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.160230978 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.621 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112898 8.b 8:d0e1dd76:::benchmark_data_smithi082_175677_object112897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.558499+0000", 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.99264867100001, 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16924559, 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.622 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112898 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558499+0000", 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.623 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558499+0000", 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558498+0000", 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3792000000000002e-05 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558562+0000", 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6529999999999999e-05 2023-10-09T04:47:02.624 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558578+0000", 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6480000000000002e-06 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.558583+0000", 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013064813 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.625 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.571648+0000", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000140407 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.571788+0000", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018515400000000001 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.571973+0000", 2023-10-09T04:47:02.626 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0065134260000000001 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.578487+0000", 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001273695 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.579760+0000", 2023-10-09T04:47:02.627 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147833726 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727594+0000", 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5462000000000003e-05 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727679+0000", 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4882999999999995e-05 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.628 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727744+0000", 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16924559 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112909 8.b 8:ddf68774:::benchmark_data_smithi082_175677_object112908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.572573+0000", 2023-10-09T04:47:02.629 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.97857468699999, 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15541010199999999, 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112909 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.630 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572573+0000", 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572573+0000", 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572571+0000", 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9944000000000007e-05 2023-10-09T04:47:02.631 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572641+0000", 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572655+0000", 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9009999999999999e-06 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.632 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.572661+0000", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023897499999999999 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.575050+0000", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011748600000000001 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.575168+0000", 2023-10-09T04:47:02.633 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000110618 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.575279+0000", 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033370850000000001 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.578616+0000", 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0013218139999999999 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.634 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.579937+0000", 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.147906697 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727844+0000", 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5640000000000001e-05 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.635 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727920+0000", 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3028999999999996e-05 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.727983+0000", 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15541010199999999 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.636 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:153729 8.2 8:489a4ae8:::benchmark_data_smithi082_175677_object36715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:01.279952+0000", 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 121.27119568800001, 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22914701400000001, 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.637 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 153729 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279952+0000", 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279952+0000", 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:02.638 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279949+0000", 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.173e-05 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279961+0000", 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2730000000000001e-05 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.639 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279984+0000", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0125e-05 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.279994+0000", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8509000000000002e-05 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.280082+0000", 2023-10-09T04:47:02.640 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000181478 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.280264+0000", 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000162354 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.280426+0000", 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0057074639999999998 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.641 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.286134+0000", 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015453299999999999 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.286288+0000", 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22263048999999999 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.642 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.643 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.508919+0000", 2023-10-09T04:47:02.643 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103473 2023-10-09T04:47:02.643 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.643 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.643 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.509022+0000", 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6476000000000004e-05 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.509099+0000", 2023-10-09T04:47:02.644 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.22914701400000001 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:153760 8.2 8:45e26d0d:::benchmark_data_smithi082_175677_object36746:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:01.344571+0000", 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 121.206576094, 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16469857099999999, 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.645 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 153760 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344571+0000", 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.646 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344571+0000", 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344569+0000", 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2469999999999997e-06 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.647 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344578+0000", 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2869000000000001e-05 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344601+0000", 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.041e-05 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344612+0000", 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8477000000000002e-05 2023-10-09T04:47:02.648 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344680+0000", 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017526799999999999 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.344855+0000", 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00022941899999999999 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.345085+0000", 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048363950000000003 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.349921+0000", 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000134411 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.650 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.350056+0000", 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15911041500000001 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.509166+0000", 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7512000000000001e-05 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.651 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.509224+0000", 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6239999999999998e-05 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.509270+0000", 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16469857099999999 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.652 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190662 8.b 8:d214fa09:::benchmark_data_smithi082_175677_object73648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.791166+0000", 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 59.759981699999997, 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20377610600000001, 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.653 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190662 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791166+0000", 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.654 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791166+0000", 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791164+0000", 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8850000000000001e-06 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791171+0000", 2023-10-09T04:47:02.655 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3703000000000001e-05 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791185+0000", 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2949999999999999e-06 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791191+0000", 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000134435 2023-10-09T04:47:02.656 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791326+0000", 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000119601 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791445+0000", 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000143638 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.657 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791589+0000", 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0082886569999999996 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.799878+0000", 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9291e-05 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.658 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.799917+0000", 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.194907161 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.994824+0000", 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3843000000000005e-05 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.659 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.994888+0000", 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3912000000000002e-05 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.994942+0000", 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20377610600000001 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.660 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190664 8.b 8:d6c995fc:::benchmark_data_smithi082_175677_object73650:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.791217+0000", 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 59.759930771000001, 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20387999200000001, 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190664 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.661 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791217+0000", 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791217+0000", 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.662 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791216+0000", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9210000000000002e-06 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791220+0000", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0319999999999998e-06 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791227+0000", 2023-10-09T04:47:02.663 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8409999999999999e-06 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.791232+0000", 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040348659999999998 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.795266+0000", 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013830299999999999 2023-10-09T04:47:02.664 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.795405+0000", 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014987099999999999 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.795555+0000", 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044611980000000004 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.665 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.800016+0000", 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048661499999999996 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804882+0000", 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19012109799999999 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.666 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995003+0000", 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0191000000000002e-05 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995043+0000", 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3387000000000002e-05 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995097+0000", 2023-10-09T04:47:02.667 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20387999200000001 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190671 8.b 8:df80b832:::benchmark_data_smithi082_175677_object73657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.806033+0000", 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 59.745114522999998, 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.189216724, 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.668 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190671 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806033+0000", 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.669 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806033+0000", 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806032+0000", 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0179999999999998e-06 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.670 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806038+0000", 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0183000000000001e-05 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806048+0000", 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5460000000000002e-06 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806052+0000", 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.6400999999999999e-05 2023-10-09T04:47:02.671 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806089+0000", 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000110207 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806199+0000", 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000114214 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.672 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806313+0000", 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048778789999999999 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.811191+0000", 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018090399999999999 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.811372+0000", 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.18378145800000001 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.673 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995153+0000", 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3439000000000001e-05 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995207+0000", 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2781999999999998e-05 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995250+0000", 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.189216724 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190672 8.b 8:d2802d6c:::benchmark_data_smithi082_175677_object73658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.818281+0000", 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 59.732866362999999, 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17709846300000001, 2023-10-09T04:47:02.675 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190672 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818281+0000", 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.676 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818281+0000", 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999971 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818278+0000", 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4997000000000001e-05 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.677 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818293+0000", 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0088999999999999e-05 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818323+0000", 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4346e-05 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.678 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818337+0000", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015602199999999999 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818493+0000", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020149 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818695+0000", 2023-10-09T04:47:02.679 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000139956 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.818835+0000", 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035899019999999999 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.822425+0000", 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00072586199999999997 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.680 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.823151+0000", 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17214548099999999 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995296+0000", 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.3875999999999999e-05 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995330+0000", 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9462000000000002e-05 2023-10-09T04:47:02.681 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.995379+0000", 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17709846300000001 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:192843 8.2 8:4d8704f2:::benchmark_data_smithi082_175677_object75829:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.682 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:06.506391+0000", 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 56.044756303, 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31807512100000002, 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 192843 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.683 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506391+0000", 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506391+0000", 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506389+0000", 2023-10-09T04:47:02.684 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8229999999999995e-06 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506397+0000", 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2863e-05 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506410+0000", 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8759999999999999e-06 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.685 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.506417+0000", 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0046744990000000004 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.511091+0000", 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000120384 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.686 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.511211+0000", 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000148176 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.511360+0000", 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0087582340000000002 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.687 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.520118+0000", 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00042753799999999998 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.520545+0000", 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.30375004 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824295+0000", 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103952 2023-10-09T04:47:02.688 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824399+0000", 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6805999999999997e-05 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824466+0000", 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31807512100000002 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.689 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:192845 8.b 8:daf45269:::benchmark_data_smithi082_175677_object75831:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:06.512324+0000", 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 56.038822973000002, 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31214410500000001, 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.690 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 192845 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512324+0000", 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.691 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512324+0000", 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512323+0000", 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0269999999999995e-06 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512331+0000", 2023-10-09T04:47:02.692 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3393e-05 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512344+0000", 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8750000000000002e-06 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.512351+0000", 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0025473140000000002 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.693 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.514898+0000", 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000279447 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.515178+0000", 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000136921 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.694 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.515315+0000", 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048225059999999998 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.520137+0000", 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0028901679999999998 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.523027+0000", 2023-10-09T04:47:02.695 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.301171682 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824199+0000", 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016052800000000001 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824360+0000", 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000108968 2023-10-09T04:47:02.696 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.824469+0000", 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31214410500000001 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:197787 8.b 8:d5d1a576:::benchmark_data_smithi082_175677_object80773:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:14.681844+0000", 2023-10-09T04:47:02.697 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 47.869303355, 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16693978400000001, 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 197787 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.698 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681844+0000", 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681844+0000", 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681842+0000", 2023-10-09T04:47:02.699 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3860000000000007e-06 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681851+0000", 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1749999999999999e-05 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681863+0000", 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6849999999999998e-06 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681868+0000", 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1693999999999997e-05 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.681920+0000", 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013176400000000001 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.701 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.682052+0000", 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013596499999999999 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.682188+0000", 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0039637439999999999 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.686151+0000", 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0058444109999999999 2023-10-09T04:47:02.702 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.691996+0000", 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15650392799999999 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.848500+0000", 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014849199999999999 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.703 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.848648+0000", 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013554100000000001 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.848784+0000", 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16693978400000001 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:197791 8.b 8:d8bb9b4e:::benchmark_data_smithi082_175677_object80777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:14.687176+0000", 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 47.863971542999998, 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.18782462699999999, 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 197791 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.705 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687176+0000", 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687176+0000", 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687174+0000", 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0900000000000001e-06 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687181+0000", 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4628e-05 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687195+0000", 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4649999999999999e-06 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687202+0000", 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5386000000000002e-05 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:02.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687247+0000", 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000122693 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687370+0000", 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013983100000000001 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.687510+0000", 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.005130374 2023-10-09T04:47:02.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.692640+0000", 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00091481600000000004 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.693555+0000", 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.18134067600000001 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.710 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.874895+0000", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8535000000000001e-05 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.874954+0000", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6496999999999999e-05 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875000+0000", 2023-10-09T04:47:02.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.18782462699999999 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:02.712 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:47:02.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:02.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:02.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:02.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:02.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:02.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:03.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:03.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:03.158 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:03.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:03.302 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:03.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:03.501 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:03.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:47:03.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:03.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:03.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:03.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:03.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:03.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:03.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:03.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:04.041 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:04 smithi082 bash[77063]: cluster 2023-10-09T04:47:02.672337+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:04.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:04.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:04.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:04.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:04.316 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:04.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:04 smithi153 bash[59565]: cluster 2023-10-09T04:47:02.672337+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:04.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:04.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:04.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:04.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:04.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:04 smithi082 bash[79733]: cluster 2023-10-09T04:47:02.672337+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 551 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 19478/26301 objects degraded (74.058%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:47:04.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:04.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:04.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:04.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:04.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:04.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:04.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:05.017 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:05.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:05.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:05.190 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:05.227 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:05 smithi082 bash[77063]: cluster 2023-10-09T04:47:04.974794+0000 mon.a (mon.0) 1462 : cluster 3 Health check update: Degraded data redundancy: 19478/26301 objects degraded (74.058%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:05 smithi082 bash[79733]: cluster 2023-10-09T04:47:04.974794+0000 mon.a (mon.0) 1462 : cluster 3 Health check update: Degraded data redundancy: 19478/26301 objects degraded (74.058%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:05.338 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:05.361 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:47:05.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:05.424 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:47:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:05 smithi153 bash[59565]: cluster 2023-10-09T04:47:04.974794+0000 mon.a (mon.0) 1462 : cluster 3 Health check update: Degraded data redundancy: 19478/26301 objects degraded (74.058%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.478 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:47:05.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:05.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:05.545 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:47:05.595 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:47:05.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.642 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:47:05.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:05.680 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:47:05.734 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.738 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:47:05.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:05.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:05.906 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.931 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:05.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:06.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:06.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:06.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:06 smithi153 bash[59565]: cluster 2023-10-09T04:47:04.672987+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 492 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 19345/23460 objects degraded (82.460%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:47:06.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:06 smithi153 bash[59565]: audit 2023-10-09T04:47:05.292767+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.82:0/4278884438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:06.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:06 smithi153 bash[59565]: audit 2023-10-09T04:47:05.582242+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.181 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:06 smithi153 bash[59565]: audit 2023-10-09T04:47:05.597984+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:06.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:06.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.361 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[77063]: cluster 2023-10-09T04:47:04.672987+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 492 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 19345/23460 objects degraded (82.460%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:47:06.361 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[77063]: audit 2023-10-09T04:47:05.292767+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.82:0/4278884438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[77063]: audit 2023-10-09T04:47:05.582242+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[77063]: audit 2023-10-09T04:47:05.597984+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[79733]: cluster 2023-10-09T04:47:04.672987+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1052: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 492 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 19345/23460 objects degraded (82.460%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[79733]: audit 2023-10-09T04:47:05.292767+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.82:0/4278884438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[79733]: audit 2023-10-09T04:47:05.582242+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.362 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:06 smithi082 bash[79733]: audit 2023-10-09T04:47:05.597984+0000 mon.a (mon.0) 1464 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:06.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:06.490 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:06.511 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:06.605 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:47:06.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:06.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:06.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:06.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:06.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:06.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:07.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:07.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:07.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:07.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: audit 2023-10-09T04:47:06.048369+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.82:0/3206629494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: audit 2023-10-09T04:47:06.049523+0000 mon.a (mon.0) 1465 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.589431+0000 mon.a (mon.0) 1466 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.594720+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: audit 2023-10-09T04:47:06.597338+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.597467+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.601948+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.603171+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: audit 2023-10-09T04:47:06.647649+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[79733]: audit 2023-10-09T04:47:06.665271+0000 mon.a (mon.0) 1470 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: audit 2023-10-09T04:47:06.048369+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.82:0/3206629494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: audit 2023-10-09T04:47:06.049523+0000 mon.a (mon.0) 1465 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.589431+0000 mon.a (mon.0) 1466 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.594720+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: audit 2023-10-09T04:47:06.597338+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.597467+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.601948+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.603171+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: audit 2023-10-09T04:47:06.647649+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:07 smithi082 bash[77063]: audit 2023-10-09T04:47:06.665271+0000 mon.a (mon.0) 1470 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:07.321 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: audit 2023-10-09T04:47:06.048369+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.82:0/3206629494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.321 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: audit 2023-10-09T04:47:06.049523+0000 mon.a (mon.0) 1465 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:07.321 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.589431+0000 mon.a (mon.0) 1466 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.594720+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: audit 2023-10-09T04:47:06.597338+0000 mon.a (mon.0) 1467 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.597467+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.601948+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.603171+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:07.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: audit 2023-10-09T04:47:06.647649+0000 mon.a (mon.0) 1469 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:07 smithi153 bash[59565]: audit 2023-10-09T04:47:06.665271+0000 mon.a (mon.0) 1470 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:07.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:07.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.485 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:07.591 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:07.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:07.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:07.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:07.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:07.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:08.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:08.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:08.125 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:08.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:08.163 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:08.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.602819+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.604237+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: cluster 2023-10-09T04:47:06.673458+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18792/17130 objects degraded (109.702%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: audit 2023-10-09T04:47:07.255906+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: audit 2023-10-09T04:47:07.257663+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:08.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:08 smithi153 bash[59565]: audit 2023-10-09T04:47:07.266127+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:08.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:08.278 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.602819+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.278 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.604237+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.278 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: cluster 2023-10-09T04:47:06.673458+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18792/17130 objects degraded (109.702%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:08.278 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: audit 2023-10-09T04:47:07.255906+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: audit 2023-10-09T04:47:07.257663+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[77063]: audit 2023-10-09T04:47:07.266127+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.602819+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.604237+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: cluster 2023-10-09T04:47:06.673458+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1054: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18792/17130 objects degraded (109.702%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: audit 2023-10-09T04:47:07.255906+0000 mon.a (mon.0) 1471 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: audit 2023-10-09T04:47:07.257663+0000 mon.a (mon.0) 1472 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:47:08.279 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:08 smithi082 bash[79733]: audit 2023-10-09T04:47:07.266127+0000 mon.a (mon.0) 1473 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:08.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:08.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:08.400 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:08.424 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:08.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:08.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:08.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:08.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:08.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:08.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:08.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:08.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:08.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:08.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:09.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:09.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:09.211 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:09.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:09.348 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:09.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:09.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:09.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:09.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:09.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:09.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:09.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:09.776 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:09.885 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:09.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:09.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:09.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:09] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:47:10.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:10.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:10.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:10.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:10.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:10 smithi153 bash[59565]: cluster 2023-10-09T04:47:08.674038+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 337 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 18792/16032 objects degraded (117.216%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:10.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:10 smithi153 bash[59565]: cluster 2023-10-09T04:47:09.975753+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 18792/16032 objects degraded (117.216%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:10.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:10.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:10.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:10.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:10 smithi082 bash[77063]: cluster 2023-10-09T04:47:08.674038+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 337 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 18792/16032 objects degraded (117.216%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:10.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:10 smithi082 bash[77063]: cluster 2023-10-09T04:47:09.975753+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 18792/16032 objects degraded (117.216%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:10.412 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:10 smithi082 bash[79733]: cluster 2023-10-09T04:47:08.674038+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1055: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 337 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 18792/16032 objects degraded (117.216%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:47:10.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:10 smithi082 bash[79733]: cluster 2023-10-09T04:47:09.975753+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 18792/16032 objects degraded (117.216%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:10.434 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:10.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:10.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:10.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:10.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:10.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:10.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:10.938 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:10.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:10.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:10.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:11.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:11.095 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:11.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:11.199 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:11.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:47:11.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:47:11.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:11.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:11.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:11.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:11.499 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:11.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:11.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:11.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:11.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:11.769 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:11.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:11.917 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:11.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:11.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:12.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:12.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:12.082 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:47:12.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:12 smithi082 bash[77063]: cluster 2023-10-09T04:47:10.674821+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 18159/8178 objects degraded (222.047%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:47:12.140 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:12 smithi082 bash[77063]: audit 2023-10-09T04:47:11.475036+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:12.140 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:12 smithi082 bash[79733]: cluster 2023-10-09T04:47:10.674821+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 18159/8178 objects degraded (222.047%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:47:12.141 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:12 smithi082 bash[79733]: audit 2023-10-09T04:47:11.475036+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:12.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:12.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:12.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:12.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:12 smithi153 bash[59565]: cluster 2023-10-09T04:47:10.674821+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1056: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 18159/8178 objects degraded (222.047%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:47:12.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:12 smithi153 bash[59565]: audit 2023-10-09T04:47:11.475036+0000 mon.a (mon.0) 1475 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:12.427 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:12.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:12.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:12.507 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:12.508 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:12.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:12.596 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:12.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:12.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:12.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:12.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:12.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:12.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:13.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:13.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:13.146 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:13.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:13.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:13.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[77063]: cluster 2023-10-09T04:47:12.071898+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[77063]: cluster 2023-10-09T04:47:12.072694+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[77063]: audit 2023-10-09T04:47:12.073034+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[77063]: cluster 2023-10-09T04:47:12.073114+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[77063]: cluster 2023-10-09T04:47:12.093442+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[79733]: cluster 2023-10-09T04:47:12.071898+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[79733]: cluster 2023-10-09T04:47:12.072694+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[79733]: audit 2023-10-09T04:47:12.073034+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:13.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[79733]: cluster 2023-10-09T04:47:12.073114+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:47:13.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:13 smithi082 bash[79733]: cluster 2023-10-09T04:47:12.093442+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:13.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:13 smithi153 bash[59565]: cluster 2023-10-09T04:47:12.071898+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:13 smithi153 bash[59565]: cluster 2023-10-09T04:47:12.072694+0000 osd.5 (osd.5) 152 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:13 smithi153 bash[59565]: audit 2023-10-09T04:47:12.073034+0000 mon.a (mon.0) 1476 : audit 1 from='client.? 172.21.15.82:0/295757205' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:13.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:13 smithi153 bash[59565]: cluster 2023-10-09T04:47:12.073114+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:47:13.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:13 smithi153 bash[59565]: cluster 2023-10-09T04:47:12.093442+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T04:47:13.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:47:13.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:13.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:13.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:13.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:13.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:13.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:13.877 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:13.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:13.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:47:13.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:14.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:14.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.152 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:14.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:14 smithi153 bash[59565]: cluster 2023-10-09T04:47:12.675297+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18159/8178 objects degraded (222.047%); 840 KiB/s, 13 objects/s recovering 2023-10-09T04:47:14.247 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 117011 objects 2023-10-09T04:47:14.247 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :183.512 2023-10-09T04:47:14.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:14.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:14 smithi082 bash[79733]: cluster 2023-10-09T04:47:12.675297+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18159/8178 objects degraded (222.047%); 840 KiB/s, 13 objects/s recovering 2023-10-09T04:47:14.393 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:14 smithi082 bash[77063]: cluster 2023-10-09T04:47:12.675297+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 173 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18159/8178 objects degraded (222.047%); 840 KiB/s, 13 objects/s recovering 2023-10-09T04:47:14.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:14.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:14.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:14.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:14.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:14.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:14.809 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:14.912 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:14.941 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:14.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:15.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:15.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:15.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:15 smithi153 bash[59565]: audit 2023-10-09T04:47:14.287969+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:15 smithi153 bash[59565]: cluster 2023-10-09T04:47:14.976842+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 18159/8178 objects degraded (222.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:15.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:15.212 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:15.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:15.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:15.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:15.464 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:15 smithi082 bash[77063]: audit 2023-10-09T04:47:14.287969+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:15.464 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:15 smithi082 bash[77063]: cluster 2023-10-09T04:47:14.976842+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 18159/8178 objects degraded (222.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:15.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:15 smithi082 bash[79733]: audit 2023-10-09T04:47:14.287969+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:15.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:15 smithi082 bash[79733]: cluster 2023-10-09T04:47:14.976842+0000 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 18159/8178 objects degraded (222.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:15.480 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:15.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:15.614 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:15.717 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:15.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:15.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:15.882 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:15.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:15.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:16.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:16.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:16.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:16.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:16 smithi153 bash[59565]: cluster 2023-10-09T04:47:14.676243+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 114 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 18045/5352 objects degraded (337.164%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:47:16.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:16.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:16 smithi082 bash[77063]: cluster 2023-10-09T04:47:14.676243+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 114 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 18045/5352 objects degraded (337.164%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:47:16.266 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:16 smithi082 bash[79733]: cluster 2023-10-09T04:47:14.676243+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 83 active+clean; 114 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 18045/5352 objects degraded (337.164%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:47:16.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:16.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:16.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:16.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:16.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:16.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:16.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:16.681 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:16.785 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:16.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:16.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:16.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:16.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:17.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:17.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:17.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:17.198 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:17 smithi153 bash[59565]: cluster 2023-10-09T04:47:16.677373+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1060: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 8.0 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 17504/249 objects degraded (7029.719%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:17.218 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:17.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:17.332 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:17 smithi082 bash[77063]: cluster 2023-10-09T04:47:16.677373+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1060: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 8.0 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 17504/249 objects degraded (7029.719%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:17.332 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:17 smithi082 bash[79733]: cluster 2023-10-09T04:47:16.677373+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1060: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 8.0 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 17504/249 objects degraded (7029.719%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:17.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:17.433 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:17.434 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:17.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:17.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:17.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:17.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:17.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:17.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:17.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:17.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:17.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:17.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:18.026 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:18.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:18.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:18.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:18.352 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:18.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:18.449 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:18.520 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:18.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:18.645 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:18.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:18.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:18.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:18.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:18.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:19.030 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:19.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:19.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:19.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:19.193 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:19.296 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:19.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:19.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:19.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:19.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:19.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:19.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:19.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:19.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:19.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:19.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:19.957 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:19 smithi082 bash[77063]: cluster 2023-10-09T04:47:18.677840+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1061: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17504/84 objects degraded (20838.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:19.958 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:19 smithi082 bash[79733]: cluster 2023-10-09T04:47:18.677840+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1061: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17504/84 objects degraded (20838.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:19.958 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:19] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-09T04:47:19.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:19.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:20.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:20.126 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:19 smithi153 bash[59565]: cluster 2023-10-09T04:47:18.677840+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1061: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 17504/84 objects degraded (20838.095%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:47:20.150 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T04:47:20.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T04:47:20.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:20.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:20.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:20.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:20.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:20.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:20.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:20.710 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:20.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.841 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:20 smithi082 bash[79733]: cluster 2023-10-09T04:47:19.978242+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 17504/84 objects degraded (20838.095%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:20.842 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:20 smithi082 bash[77063]: cluster 2023-10-09T04:47:19.978242+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 17504/84 objects degraded (20838.095%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:20.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:20.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:20.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:20.992 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:20 smithi153 bash[59565]: cluster 2023-10-09T04:47:19.978242+0000 mon.a (mon.0) 1480 : cluster 3 Health check update: Degraded data redundancy: 17504/84 objects degraded (20838.095%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:20.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:21.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:21.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:21.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:21.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:21.341 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:21.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:21.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:21.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:21.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:21.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:21.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:21.759 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:21.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:21.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:21.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:21.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:21.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:22.001 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:21 smithi082 bash[79733]: cluster 2023-10-09T04:47:20.678949+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1062: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 17504/84 objects degraded (20838.095%); 729 KiB/s, 47 objects/s recovering 2023-10-09T04:47:22.002 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:21 smithi082 bash[77063]: cluster 2023-10-09T04:47:20.678949+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1062: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 17504/84 objects degraded (20838.095%); 729 KiB/s, 47 objects/s recovering 2023-10-09T04:47:22.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:22.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:22.130 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:21 smithi153 bash[59565]: cluster 2023-10-09T04:47:20.678949+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1062: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 17504/84 objects degraded (20838.095%); 729 KiB/s, 47 objects/s recovering 2023-10-09T04:47:22.146 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:22.181 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:22.181 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:22.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:22.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:22.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:22.381 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:22.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:22.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:22.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:22.532 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:22.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:22.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:22.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:22.794 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:22.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:22.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:22.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:23.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:23.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:23.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:23.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:23.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:47:23.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:23.307 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:23.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:23.411 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:23.542 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.573 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:23.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:23.702 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:23.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:23.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.837 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:23.941 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:23.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:23.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:23.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:24.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:24.079 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:23 smithi153 bash[59565]: cluster 2023-10-09T04:47:22.679472+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1063: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 17504/84 objects degraded (20838.095%); 688 KiB/s, 44 objects/s recovering 2023-10-09T04:47:24.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:24.203 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:24.211 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:23 smithi082 bash[77063]: cluster 2023-10-09T04:47:22.679472+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1063: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 17504/84 objects degraded (20838.095%); 688 KiB/s, 44 objects/s recovering 2023-10-09T04:47:24.211 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:23 smithi082 bash[79733]: cluster 2023-10-09T04:47:22.679472+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1063: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 17504/84 objects degraded (20838.095%); 688 KiB/s, 44 objects/s recovering 2023-10-09T04:47:24.223 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:24.327 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:24.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:24.487 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:24.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:24.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:24.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:24.879 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:24.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:24.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:24.914 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:24.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:25.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.126 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:25.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.247 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:25.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:25.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:25.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:25.491 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:25.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:25.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:25.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:25.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:25.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:25.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:25.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:25.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:25.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:25.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:26.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:26.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:26.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:26.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:25 smithi082 bash[77063]: cluster 2023-10-09T04:47:24.679975+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1064: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 17504/84 objects degraded (20838.095%); 608 KiB/s, 76 objects/s recovering 2023-10-09T04:47:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:25 smithi082 bash[79733]: cluster 2023-10-09T04:47:24.679975+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1064: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 17504/84 objects degraded (20838.095%); 608 KiB/s, 76 objects/s recovering 2023-10-09T04:47:26.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.256 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:25 smithi153 bash[59565]: cluster 2023-10-09T04:47:24.679975+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1064: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 17504/84 objects degraded (20838.095%); 608 KiB/s, 76 objects/s recovering 2023-10-09T04:47:26.275 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:26.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:26.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:26.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:26.503 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:26.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:26.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:26.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:26.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:26.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:26.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:27.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:27.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:27.183 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:27.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:27.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:27.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:27.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:27.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:27.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:27.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:27.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:27.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:27.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:27.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:27.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:28.002 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:28.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:28.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:28.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:28.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:27 smithi082 bash[77063]: cluster 2023-10-09T04:47:26.680748+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1065: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:28.244 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:28 smithi082 bash[79733]: cluster 2023-10-09T04:47:26.680748+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1065: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:28.342 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.356 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:27 smithi153 bash[59565]: cluster 2023-10-09T04:47:26.680748+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1065: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:28.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:28.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:28.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:28.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:28.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:28.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:28.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:28.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:28.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:28.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:28.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:29.125 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:29.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:29.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:29.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:29.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:29.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:29.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:29.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:29.780 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:29.809 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:29.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:29.920 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:29] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:47:29.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:29.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:30.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:30.048 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:29 smithi153 bash[59565]: cluster 2023-10-09T04:47:28.681282+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1066: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:30.048 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:29 smithi153 bash[59565]: audit 2023-10-09T04:47:29.288533+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:30.066 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:30.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:30.180 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:29 smithi082 bash[77063]: cluster 2023-10-09T04:47:28.681282+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1066: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:30.181 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:29 smithi082 bash[77063]: audit 2023-10-09T04:47:29.288533+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:30.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:29 smithi082 bash[79733]: cluster 2023-10-09T04:47:28.681282+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1066: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 17504/84 objects degraded (20838.095%); 0 B/s, 66 objects/s recovering 2023-10-09T04:47:30.181 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:29 smithi082 bash[79733]: audit 2023-10-09T04:47:29.288533+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:30.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:30.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:30.328 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:30.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:30.558 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:30.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:30.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:30.796 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:30.822 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:30.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:30.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:30.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:31.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:31.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:31.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:31.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:31.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:31.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:31.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:31.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:31.529 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:31.529 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:31.584 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:31.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:31.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:31.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:31.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:31.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:31.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:31.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:31.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:32.093 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:32.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:32.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:32 smithi082 bash[77063]: cluster 2023-10-09T04:47:30.682089+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1067: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 106 objects/s recovering 2023-10-09T04:47:32.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:32 smithi082 bash[79733]: cluster 2023-10-09T04:47:30.682089+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1067: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 106 objects/s recovering 2023-10-09T04:47:32.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:32.355 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:32.366 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:32 smithi153 bash[59565]: cluster 2023-10-09T04:47:30.682089+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1067: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 106 objects/s recovering 2023-10-09T04:47:32.387 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:32.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:47:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:32.490 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:32.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:32.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:32.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:32.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:32.878 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.902 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:32.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:33.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:33.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:33.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:33.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:33.308 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:33.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:33.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:33.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:33.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:33.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:33.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:33.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:34.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:34.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:34.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:34.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:34.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:34.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:34 smithi082 bash[77063]: cluster 2023-10-09T04:47:32.682639+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1068: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 76 objects/s recovering 2023-10-09T04:47:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:34 smithi082 bash[79733]: cluster 2023-10-09T04:47:32.682639+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1068: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 76 objects/s recovering 2023-10-09T04:47:34.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:34.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:34 smithi153 bash[59565]: cluster 2023-10-09T04:47:32.682639+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1068: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 76 objects/s recovering 2023-10-09T04:47:34.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:34.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:34.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:34.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:34.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:34.730 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:34.833 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:34.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:34.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:34.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:35.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:35.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:35.247 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:35.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:35.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:35.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:35.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:35.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:35.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:35.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:35.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:35.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:35.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:35.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:35.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:36.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:36.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:36.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:36 smithi082 bash[77063]: cluster 2023-10-09T04:47:34.683371+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1069: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 119 objects/s recovering 2023-10-09T04:47:36.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:36 smithi082 bash[79733]: cluster 2023-10-09T04:47:34.683371+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1069: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 119 objects/s recovering 2023-10-09T04:47:36.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:36.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:36.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:36 smithi153 bash[59565]: cluster 2023-10-09T04:47:34.683371+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1069: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 119 objects/s recovering 2023-10-09T04:47:36.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:36.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:36.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:36.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:36.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:36.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:36.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:36.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:36.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:37.080 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.099 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:37.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:37.211 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:37.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:37.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:37.356 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:37.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:37.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:37.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:37.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:37.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:37.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:37.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:37.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:37.983 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:38.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:38.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:38.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:38.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:38.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.270 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.271 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:38 smithi153 bash[59565]: cluster 2023-10-09T04:47:36.684525+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1070: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:38.373 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:38.402 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:38.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:38.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:38 smithi082 bash[79733]: cluster 2023-10-09T04:47:36.684525+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1070: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:38 smithi082 bash[77063]: cluster 2023-10-09T04:47:36.684525+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1070: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:38.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:38.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:38.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:38.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:38.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:38.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:38.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:38.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:39.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:39.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:39.161 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:39.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:39.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:39.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:39.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:39.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:39.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:39.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:39.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:39.662 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:47:39.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:39.685 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:47:39.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:39.712 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:47:39.754 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:47:39.792 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:47:39.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:39.814 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:47:39.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:39.851 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:47:39.882 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:47:39.884 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.884 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:47:39.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:39.962 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:39] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-09T04:47:39.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:39.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:40.016 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[79733]: cluster 2023-10-09T04:47:38.685118+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1071: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:40.016 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[79733]: audit 2023-10-09T04:47:39.632634+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.82:0/1865597942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:40.016 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[79733]: audit 2023-10-09T04:47:39.790486+0000 mon.a (mon.0) 1482 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:40.016 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[79733]: audit 2023-10-09T04:47:39.971542+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:47:40.038 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:47:40.039 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:47:40.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:40.191 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:40.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[77063]: cluster 2023-10-09T04:47:38.685118+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1071: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[77063]: audit 2023-10-09T04:47:39.632634+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.82:0/1865597942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:40.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[77063]: audit 2023-10-09T04:47:39.790486+0000 mon.a (mon.0) 1482 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:40.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:40 smithi082 bash[77063]: audit 2023-10-09T04:47:39.971542+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:47:40.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:40 smithi153 bash[59565]: cluster 2023-10-09T04:47:38.685118+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1071: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17504/84 objects degraded (20838.095%); 0 B/s, 82 objects/s recovering 2023-10-09T04:47:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:40 smithi153 bash[59565]: audit 2023-10-09T04:47:39.632634+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.82:0/1865597942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:40 smithi153 bash[59565]: audit 2023-10-09T04:47:39.790486+0000 mon.a (mon.0) 1482 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:40 smithi153 bash[59565]: audit 2023-10-09T04:47:39.971542+0000 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:47:40.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:40.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:40.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:40.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:40.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:40.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:40.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:40.774 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:47:40.774 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:47:40.774 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T04:47:40.775 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:47:40.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:40.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:40.966 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:40.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:40.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:41.050 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_3' removed 2023-10-09T04:47:41.097 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:41.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.013035+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:41.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.021624+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: audit 2023-10-09T04:47:40.025322+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:41.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: audit 2023-10-09T04:47:40.025686+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:47:41.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.025799+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:47:41.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.026090+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.115350+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:41 smithi153 bash[59565]: audit 2023-10-09T04:47:40.172654+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:47:41.126 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.013035+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:41.126 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.021624+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: audit 2023-10-09T04:47:40.025322+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: audit 2023-10-09T04:47:40.025686+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.025799+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.026090+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.115350+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.127 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[77063]: audit 2023-10-09T04:47:40.172654+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:47:41.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.013035+0000 mon.a (mon.0) 1484 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.021624+0000 osd.5 (osd.5) 153 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: audit 2023-10-09T04:47:40.025322+0000 mon.a (mon.0) 1485 : audit 1 from='client.? 172.21.15.82:0/3705134301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: audit 2023-10-09T04:47:40.025686+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.82:0/2567237859' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.025799+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.026090+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.115350+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:41.128 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:41 smithi082 bash[79733]: audit 2023-10-09T04:47:40.172654+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:47:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:41.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:41.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:41.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:41.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:41.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:41.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:41.501 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:41.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:41.632 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:41.735 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:41.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:41.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:41.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:47:41.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:41.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:41.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:42.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:42.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.114818+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:40.686057+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1073: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17152/84 objects degraded (20419.048%); 0 B/s, 107 objects/s recovering 2023-10-09T04:47:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.020803+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: audit 2023-10-09T04:47:41.026693+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:47:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.026766+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:47:42.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.026882+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.105789+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.107660+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.109344+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:42 smithi153 bash[59565]: cluster 2023-10-09T04:47:41.201672+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.274 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.114818+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:42.274 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:40.686057+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1073: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17152/84 objects degraded (20419.048%); 0 B/s, 107 objects/s recovering 2023-10-09T04:47:42.274 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.020803+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:42.274 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: audit 2023-10-09T04:47:41.026693+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.026766+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.026882+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.105789+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.107660+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.109344+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[79733]: cluster 2023-10-09T04:47:41.201672+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.114818+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:40.686057+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1073: 89 pgs: 3 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 17152/84 objects degraded (20419.048%); 0 B/s, 107 objects/s recovering 2023-10-09T04:47:42.275 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.020803+0000 mon.a (mon.0) 1489 : cluster 3 Health check update: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: audit 2023-10-09T04:47:41.026693+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.82:0/569641953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.026766+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.026882+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.105789+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.107660+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.109344+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:42 smithi082 bash[77063]: cluster 2023-10-09T04:47:41.201672+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T04:47:42.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:42.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:42.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:42.523 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:42.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:42.656 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:42.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:42.794 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:42.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:42.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:42.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.961 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:42.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:43.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:43.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:43.195 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:43.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:43.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:43 smithi082 bash[77063]: cluster 2023-10-09T04:47:43.017179+0000 mon.a (mon.0) 1492 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized) 2023-10-09T04:47:43.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:43 smithi082 bash[79733]: cluster 2023-10-09T04:47:43.017179+0000 mon.a (mon.0) 1492 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized) 2023-10-09T04:47:43.267 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:47:43.268 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:47:43.268 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T04:47:43.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T04:47:43.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:47:43.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:43.343 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:43 smithi153 bash[59565]: cluster 2023-10-09T04:47:43.017179+0000 mon.a (mon.0) 1492 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17152/84 objects degraded (20419.048%), 3 pgs degraded, 6 pgs undersized) 2023-10-09T04:47:43.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:43.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:43.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:43.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:43.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:43.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:43.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:43.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:43.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:43.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:43.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:43.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:44.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:44.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:44.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:44.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:44.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:44 smithi082 bash[77063]: cluster 2023-10-09T04:47:42.686688+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:44.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:44 smithi082 bash[79733]: cluster 2023-10-09T04:47:42.686688+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:44.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:44.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:44.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:44.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:44.419 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:44 smithi153 bash[59565]: cluster 2023-10-09T04:47:42.686688+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:44.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:44.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:44.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:44.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:44.706 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:44.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:44.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:44.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:44.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:44.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:44.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:44.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:45.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:45.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:47:45.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:45.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:47:45.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:45.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[77063]: audit 2023-10-09T04:47:44.296692+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:45.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[77063]: audit 2023-10-09T04:47:44.298383+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:45.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[77063]: cluster 2023-10-09T04:47:44.687483+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:45.371 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[79733]: audit 2023-10-09T04:47:44.296692+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:45.371 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[79733]: audit 2023-10-09T04:47:44.298383+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:45.372 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:45 smithi082 bash[79733]: cluster 2023-10-09T04:47:44.687483+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:45.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:45.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:45.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:45.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:45 smithi153 bash[59565]: audit 2023-10-09T04:47:44.296692+0000 mon.a (mon.0) 1493 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:45 smithi153 bash[59565]: audit 2023-10-09T04:47:44.298383+0000 mon.a (mon.0) 1494 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:45.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:45 smithi153 bash[59565]: cluster 2023-10-09T04:47:44.687483+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:45.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:45.809 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:45.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:45.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:45.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:46.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:46.081 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:46.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:46.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:46.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:46.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:46.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:46.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:46.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.627 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:46.672 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:46.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:46.760 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:46.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:46.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:46.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:47.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:47.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:47.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:47.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:47.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:47.298 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:47.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:47.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:47.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:47.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:47.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:47.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:47.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:47.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:47.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:47.822 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:47 smithi082 bash[77063]: cluster 2023-10-09T04:47:46.688445+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:47.822 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:47 smithi082 bash[79733]: cluster 2023-10-09T04:47:46.688445+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:47.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:47.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:47.954 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:47 smithi153 bash[59565]: cluster 2023-10-09T04:47:46.688445+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:47.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:47.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:48.075 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:48.076 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:48.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:48.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:48.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:48.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:48.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:48.342 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:48.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:48.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:48.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:48.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:48.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:48.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:48.783 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:48.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:48.921 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:48.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:49.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:49.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:49.094 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:49.094 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:49.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:49.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:49.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:49.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:49.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:49.405 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:49.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:49.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:49.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:49.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:49.701 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:49.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:49.806 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:49.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:49.850 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:49 smithi082 bash[77063]: cluster 2023-10-09T04:47:48.689143+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:49.851 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:49] "GET /metrics HTTP/1.1" 200 34096 "" "Prometheus/2.33.4" 2023-10-09T04:47:49.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:49.953 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:49 smithi153 bash[59565]: cluster 2023-10-09T04:47:48.689143+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:49.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:49.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:50.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:50.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:50.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:50.200 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:50.272 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:49 smithi082 bash[79733]: cluster 2023-10-09T04:47:48.689143+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:50.283 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:47:50.283 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:47:50.284 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:47:50.284 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:47:50.284 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:47:50.330 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:50.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:50.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:50.485 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:50.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:50.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:50.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:50.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:50.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:50.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:50.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:50.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:51.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:51.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:51.132 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:50 smithi082 bash[77063]: audit 2023-10-09T04:47:49.883009+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:51.133 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:50 smithi082 bash[79733]: audit 2023-10-09T04:47:49.883009+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:51.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:50 smithi153 bash[59565]: audit 2023-10-09T04:47:49.883009+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:47:51.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:51.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:47:51.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:51.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:51.393 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:51.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:51.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:51.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:51.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:51.881 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:51.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:51.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:52.014 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:52.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:52.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:52.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:51 smithi153 bash[59565]: cluster 2023-10-09T04:47:50.689729+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:52.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:52.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:52 smithi082 bash[77063]: cluster 2023-10-09T04:47:50.689729+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:52.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:52 smithi082 bash[79733]: cluster 2023-10-09T04:47:50.689729+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:52.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:52.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:52.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:52.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.576 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:52.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:52.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:52.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:52.949 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:52.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:52.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:53.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:53.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:53.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:53.234 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:53.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:53.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:53.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:53.491 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:53.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:53.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:53.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:53.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:53.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:53.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:53.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:53.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:54.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:54.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:54.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:53 smithi153 bash[59565]: cluster 2023-10-09T04:47:52.690158+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:54.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.255 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:53 smithi082 bash[77063]: cluster 2023-10-09T04:47:52.690158+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:54.256 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:53 smithi082 bash[79733]: cluster 2023-10-09T04:47:52.690158+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:54.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:54.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:54.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:54.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:54.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:54.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:54.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:54.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:54.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:54.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:54.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:55.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:55.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:55.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:55.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:55.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.345 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:55.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:55.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:55.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:55.614 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:55.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:55.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:55.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:55.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:55.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:56.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:56.033 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:56.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.149 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:56.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:56 smithi082 bash[77063]: cluster 2023-10-09T04:47:54.690669+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:56.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:56 smithi082 bash[79733]: cluster 2023-10-09T04:47:54.690669+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:56.253 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:56.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:56.381 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:56.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:56 smithi153 bash[59565]: cluster 2023-10-09T04:47:54.690669+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:56.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:56.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:56.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:56.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:56.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:56.647 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:56.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:56.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.813 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:56.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:56.950 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:56.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:56.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:57.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:57.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:57.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:57.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:57.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:57.346 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:57.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:57.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:57.478 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:57.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:57.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:57.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:57.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:57.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:57.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:57.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:57.987 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:58.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:58.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:58.148 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:58.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:47:58 smithi153 bash[59565]: cluster 2023-10-09T04:47:56.691245+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:58.252 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:58.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:58.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:58.392 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:47:58 smithi082 bash[77063]: cluster 2023-10-09T04:47:56.691245+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:58.392 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:47:58 smithi082 bash[79733]: cluster 2023-10-09T04:47:56.691245+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:47:58.405 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:58.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:58.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:58.636 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:58.669 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:58.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:58.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:58.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:58.911 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:58.945 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:58.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:59.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.080 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:59.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:59.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:59.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:59.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:59.578 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:59.606 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:59.647 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:47:59.647 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:47:59.647 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:47:59.647 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:47:59.647 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:144268 7.c 7:33ea9919:::benchmark_data_smithi082_155217_object41608:head [delete] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:40:11.012735+0000", 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 468.62765748300001, 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0957939479999999, 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 144268 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.648 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012735+0000", 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012735+0000", 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.649 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012734+0000", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2660000000000001e-06 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012742+0000", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2675e-05 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012755+0000", 2023-10-09T04:47:59.650 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2960000000000004e-06 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012761+0000", 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4838000000000005e-05 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012826+0000", 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1317e-05 2023-10-09T04:47:59.651 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.012837+0000", 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.78959484000000002 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802432+0000", 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4834999999999999e-05 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.652 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802457+0000", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.222697369 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.025154+0000", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013670699999999999 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.025291+0000", 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00071936099999999996 2023-10-09T04:47:59.653 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.026010+0000", 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.018339193 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.654 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044349+0000", 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0410000000000003e-06 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044358+0000", 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.010882179000000001 2023-10-09T04:47:59.655 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.055241+0000", 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.053113529999999999 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.656 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.108354+0000", 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1693999999999995e-05 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.108426+0000", 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000103149 2023-10-09T04:47:59.657 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.108529+0000", 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0957939479999999 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.658 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:144275 7.c 7:3e1f62de:::benchmark_data_smithi082_155217_object41615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:40:11.021821+0000", 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 468.618571804, 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0897992269999999, 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.659 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 144275 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.660 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021821+0000", 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021821+0000", 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:59.661 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021820+0000", 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7049999999999998e-06 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.662 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021825+0000", 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1689999999999998e-06 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021834+0000", 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1149999999999999e-06 2023-10-09T04:47:59.663 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021840+0000", 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9478000000000001e-05 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.664 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021909+0000", 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1239e-05 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.021921+0000", 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.780939787 2023-10-09T04:47:59.665 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802860+0000", 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2529e-05 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802873+0000", 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2324392479999999 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.666 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.035312+0000", 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014475999999999999 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.035457+0000", 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016914400000000001 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.667 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.035626+0000", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0087386570000000004 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044365+0000", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5157000000000002e-05 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044400+0000", 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014681179000000001 2023-10-09T04:47:59.668 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.059081+0000", 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.052408416999999999 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111489+0000", 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4588000000000001e-05 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.669 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111534+0000", 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5964999999999994e-05 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111620+0000", 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0897992269999999 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.670 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35259.0:144279 7.c 7:39326d43:::benchmark_data_smithi082_155217_object41619:head [delete] snapc 0=[] ondisk+write+known_if_redirected e189)", 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:40:11.031151+0000", 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 468.60924113099998, 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0805982489999999, 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35259", 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/278359728", 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 144279 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.671 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031151+0000", 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.672 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031151+0000", 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031149+0000", 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.098e-06 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.673 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031158+0000", 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6634e-05 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031175+0000", 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1544e-05 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.674 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031187+0000", 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6552999999999998e-05 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031253+0000", 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2427e-05 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.031266+0000", 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.77164392599999998 2023-10-09T04:47:59.675 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802910+0000", 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1579e-05 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:11.802921+0000", 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.236973154 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.676 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.039894+0000", 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000151015 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.040045+0000", 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020078899999999999 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T04:47:59.677 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.040246+0000", 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0045132499999999999 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044759+0000", 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6960000000000005e-06 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.044767+0000", 2023-10-09T04:47:59.678 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014356727 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.059124+0000", 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.052531942999999998 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.679 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111656+0000", 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.7997999999999999e-05 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111684+0000", 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5971999999999998e-05 2023-10-09T04:47:59.680 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:40:13.111750+0000", 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0805982489999999 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.681 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:7955 8.9 8:9d3096c6:::benchmark_data_smithi082_175677_object7954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:42:42.340481+0000", 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 317.29991149699998, 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42219989000000002, 2023-10-09T04:47:59.682 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 7955 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.683 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340481+0000", 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.684 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340481+0000", 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340480+0000", 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7037000000000001e-05 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.685 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340557+0000", 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2539e-05 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.686 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340569+0000", 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6830000000000008e-06 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.340577+0000", 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00062166299999999995 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.687 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.341199+0000", 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015479400000000001 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.688 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.341354+0000", 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000171962 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.341526+0000", 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00055527300000000001 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.689 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.342081+0000", 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034041029999999999 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.345485+0000", 2023-10-09T04:47:59.690 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.41681089799999999 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.762296+0000", 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5091000000000004e-05 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.691 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.762361+0000", 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00032002899999999999 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:42:42.762681+0000", 2023-10-09T04:47:59.692 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42219989000000002 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:120510 8.9 8:91e4419a:::benchmark_data_smithi082_175677_object3496:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T04:47:59.693 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:14.651347+0000", 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 224.98904517700001, 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.112417603, 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.694 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 120510 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651347+0000", 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.695 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651347+0000", 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.696 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651346+0000", 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1579999999999999e-06 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651353+0000", 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4029000000000001e-05 2023-10-09T04:47:59.697 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651367+0000", 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.754e-06 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.698 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651374+0000", 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014646799999999999 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651520+0000", 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00029430599999999998 2023-10-09T04:47:59.699 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.651815+0000", 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000420402 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.652235+0000", 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0051560080000000001 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.700 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.657391+0000", 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0077035489999999996 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.665095+0000", 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.098556609000000003 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.763651+0000", 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7630000000000001e-05 2023-10-09T04:47:59.701 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.763719+0000", 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6208000000000002e-05 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:14.763765+0000", 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.112417603 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.702 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:127137 8.9 8:9b2d490a:::benchmark_data_smithi082_175677_object10123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:22.530311+0000", 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 217.11008117700001, 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083473698999999998, 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.703 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 127137 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530311+0000", 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530311+0000", 2023-10-09T04:47:59.704 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530310+0000", 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7390000000000002e-06 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530317+0000", 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9549999999999992e-06 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.705 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530325+0000", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8720000000000001e-06 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530330+0000", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9438999999999999e-05 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530379+0000", 2023-10-09T04:47:59.706 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000113503 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530493+0000", 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000139302 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.530632+0000", 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0045113799999999997 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.707 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.535143+0000", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035606240000000001 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.538704+0000", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.074977630000000003 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.613682+0000", 2023-10-09T04:47:59.708 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8003e-05 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.613740+0000", 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5525000000000002e-05 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:22.613785+0000", 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.083473698999999998 2023-10-09T04:47:59.709 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:127900 8.5 8:a282f31c:::benchmark_data_smithi082_175677_object10886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:23.515412+0000", 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 216.12498095000001, 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.070831917999999994, 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.710 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 127900 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515412+0000", 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.711 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515412+0000", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515409+0000", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.249e-05 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515432+0000", 2023-10-09T04:47:59.712 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1186999999999999e-05 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515453+0000", 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0064999999999999e-05 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.713 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.515463+0000", 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.01811958 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.533583+0000", 2023-10-09T04:47:59.714 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000118872 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.533702+0000", 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011996400000000001 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.715 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.533822+0000", 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.015487135000000001 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.549309+0000", 2023-10-09T04:47:59.716 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5810000000000006e-05 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.549404+0000", 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.036744048000000001 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.717 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.586149+0000", 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4821000000000002e-05 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.586203+0000", 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0170000000000003e-05 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.718 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:23.586244+0000", 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.070831917999999994 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:128840 8.9 8:92de59ba:::benchmark_data_smithi082_175677_object11826:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:24.685443+0000", 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 214.954949191, 2023-10-09T04:47:59.719 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095608026999999998, 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128840 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685443+0000", 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.720 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685443+0000", 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685442+0000", 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2019999999999993e-06 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.721 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685451+0000", 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2879e-05 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685464+0000", 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6780000000000006e-06 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.722 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685473+0000", 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4313e-05 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685527+0000", 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000145943 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685673+0000", 2023-10-09T04:47:59.723 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014859000000000001 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.685821+0000", 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0081250449999999991 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.693946+0000", 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023040600000000001 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.724 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.696251+0000", 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.084623085000000001 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.780874+0000", 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000136546 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.725 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.781010+0000", 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1190999999999999e-05 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.781051+0000", 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095608026999999998 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.726 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:128842 8.9 8:9986f193:::benchmark_data_smithi082_175677_object11828:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:24.687736+0000", 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 214.95265661100001, 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.103627367, 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128842 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.727 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687736+0000", 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687736+0000", 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.728 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687734+0000", 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7930000000000003e-06 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687743+0000", 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7730000000000001e-05 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687761+0000", 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6809999999999998e-06 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.729 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.687768+0000", 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020891109999999998 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.689857+0000", 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000125974 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.730 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.689983+0000", 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7484999999999993e-05 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.690071+0000", 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040201569999999999 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.694091+0000", 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0068554690000000003 2023-10-09T04:47:59.731 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.732 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.732 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.733 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.700946+0000", 2023-10-09T04:47:59.733 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090309233000000003 2023-10-09T04:47:59.733 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.733 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.733 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.791256+0000", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8359000000000001e-05 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.791314+0000", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9230999999999997e-05 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:24.791363+0000", 2023-10-09T04:47:59.734 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.103627367 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:153881 8.9 8:9181288e:::benchmark_data_smithi082_175677_object36867:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:01.696099+0000", 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 177.94429360399999, 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15490895299999999, 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.735 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 153881 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696099+0000", 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.736 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696099+0000", 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696096+0000", 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2996000000000001e-05 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696109+0000", 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4848e-05 2023-10-09T04:47:59.737 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696134+0000", 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7890000000000006e-06 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696144+0000", 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9923999999999998e-05 2023-10-09T04:47:59.738 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696204+0000", 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014071 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696345+0000", 2023-10-09T04:47:59.739 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014101900000000001 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.696486+0000", 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0055095550000000002 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.740 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.701995+0000", 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00027044599999999998 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.702266+0000", 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14867639999999999 2023-10-09T04:47:59.741 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.850942+0000", 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7382000000000002e-05 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.742 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.850979+0000", 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.8452000000000001e-05 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:01.851008+0000", 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15490895299999999 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.743 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:159297 8.9 8:98a178e3:::benchmark_data_smithi082_175677_object42283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:10.688760+0000", 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 168.95163224500001, 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.110996709, 2023-10-09T04:47:59.744 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 159297 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.745 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688760+0000", 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688760+0000", 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688759+0000", 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7850000000000003e-06 2023-10-09T04:47:59.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688766+0000", 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7484000000000001e-05 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.748 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688784+0000", 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2980000000000003e-06 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688791+0000", 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3365000000000002e-05 2023-10-09T04:47:59.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688844+0000", 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000138825 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.688983+0000", 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000200371 2023-10-09T04:47:59.750 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.689183+0000", 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.012315779000000001 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.751 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.701499+0000", 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0071483279999999998 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708648+0000", 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090991706000000006 2023-10-09T04:47:59.752 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.799639+0000", 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3353999999999998e-05 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.753 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.799693+0000", 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4443000000000006e-05 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.799757+0000", 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.110996709 2023-10-09T04:47:59.754 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:159306 8.9 8:92740dc4:::benchmark_data_smithi082_175677_object42292:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:10.708165+0000", 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 168.932227235, 2023-10-09T04:47:59.755 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091915177000000001, 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 159306 2023-10-09T04:47:59.756 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708165+0000", 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.757 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708165+0000", 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708163+0000", 2023-10-09T04:47:59.758 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7110000000000007e-06 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708173+0000", 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3709e-05 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.759 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708187+0000", 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.793e-06 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708194+0000", 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047944099999999999 2023-10-09T04:47:59.760 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708673+0000", 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000126375 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.761 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708799+0000", 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000145103 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.708945+0000", 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0042983309999999999 2023-10-09T04:47:59.762 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.713243+0000", 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0013720399999999999 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.763 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714615+0000", 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.085214235999999999 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.799829+0000", 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000115894 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.764 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.799945+0000", 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000135382 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.765 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.800080+0000", 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.091915177000000001 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.766 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:159312 8.9 8:9edcd4fd:::benchmark_data_smithi082_175677_object42298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:10.714550+0000", 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 168.92584274199999, 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090819514000000004, 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.767 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 159312 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714550+0000", 2023-10-09T04:47:59.768 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714550+0000", 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.769 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714549+0000", 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5470000000000002e-06 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714555+0000", 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4119000000000001e-05 2023-10-09T04:47:59.770 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714579+0000", 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5630000000000003e-06 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.771 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714587+0000", 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2312999999999999e-05 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714639+0000", 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000119444 2023-10-09T04:47:59.772 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714759+0000", 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015672599999999999 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.773 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.714915+0000", 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0050267389999999997 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.719942+0000", 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000203704 2023-10-09T04:47:59.774 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.720146+0000", 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.085077600000000003 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.775 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.805223+0000", 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3298000000000001e-05 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.805307+0000", 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2706000000000006e-05 2023-10-09T04:47:59.776 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:10.805369+0000", 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090819514000000004 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.777 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:185907 8.9 8:9d37994d:::benchmark_data_smithi082_175677_object68893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:54.667868+0000", 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 124.972524089, 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090042260999999998, 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.778 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 185907 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.779 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667868+0000", 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667868+0000", 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:59.780 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667866+0000", 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1909000000000001e-05 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.781 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667878+0000", 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7968000000000001e-05 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667896+0000", 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:47:59.782 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667904+0000", 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9658999999999999e-05 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.783 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.667963+0000", 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017118299999999999 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.668134+0000", 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016911899999999999 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.784 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.668304+0000", 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0043745650000000004 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.672678+0000", 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0085122600000000007 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.785 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.681190+0000", 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.076624740999999996 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.757815+0000", 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8908e-05 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.757864+0000", 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6706000000000003e-05 2023-10-09T04:47:59.786 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:54.757911+0000", 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.090042260999999998 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:192648 8.9 8:9abd252f:::benchmark_data_smithi082_175677_object75634:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:06.151085+0000", 2023-10-09T04:47:59.787 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 113.489307926, 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.070669963000000002, 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 192648 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.788 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151085+0000", 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151085+0000", 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151082+0000", 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0188e-05 2023-10-09T04:47:59.789 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151093+0000", 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1882000000000002e-05 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.790 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151114+0000", 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8709999999999992e-06 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151124+0000", 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9252999999999997e-05 2023-10-09T04:47:59.791 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151184+0000", 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000175248 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.792 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151359+0000", 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024746099999999998 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.151606+0000", 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048966929999999997 2023-10-09T04:47:59.793 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.156503+0000", 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044404860000000004 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.794 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.160943+0000", 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.060709574000000002 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.221653+0000", 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5519999999999997e-05 2023-10-09T04:47:59.795 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.221708+0000", 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6088000000000003e-05 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.796 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.221755+0000", 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.070669963000000002 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:192852 8.9 8:9cddcae4:::benchmark_data_smithi082_175677_object75838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:47:59.797 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 113.11800913899999, 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31415913600000001, 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 192852 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.798 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522382+0000", 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7500000000000003e-06 2023-10-09T04:47:59.799 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522390+0000", 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.377e-05 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522403+0000", 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9600000000000006e-06 2023-10-09T04:47:59.800 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522412+0000", 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8878999999999998e-05 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522451+0000", 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7504999999999996e-05 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.801 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522549+0000", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7978000000000003e-05 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522647+0000", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040000080000000002 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.526647+0000", 2023-10-09T04:47:59.802 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00037852200000000001 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.527025+0000", 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.30938406600000001 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836409+0000", 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2324999999999995e-05 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.803 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836492+0000", 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0847999999999998e-05 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836543+0000", 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31415913600000001 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.804 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:217008 8.9 8:99eb6c3d:::benchmark_data_smithi082_175677_object99994:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:46.254740+0000", 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 73.385652997999998, 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095889136999999999, 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 217008 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.805 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254740+0000", 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254740+0000", 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254738+0000", 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3999999999999998e-06 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254747+0000", 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6932999999999999e-05 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254764+0000", 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7579999999999992e-06 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.254772+0000", 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00136838 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.256140+0000", 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013996899999999999 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.256280+0000", 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000140927 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.256421+0000", 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013524071 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.809 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.269945+0000", 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0047029359999999996 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.274648+0000", 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.075817139000000006 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.350465+0000", 2023-10-09T04:47:59.810 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4825000000000001e-05 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.350560+0000", 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8786000000000002e-05 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.811 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.350629+0000", 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.095889136999999999 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.812 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:217012 8.9 8:9c3310ec:::benchmark_data_smithi082_175677_object99998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:46.258276+0000", 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 73.382116338000003, 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.092792560999999996, 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.813 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 217012 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258276+0000", 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.814 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258276+0000", 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.815 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258275+0000", 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0114e-05 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258285+0000", 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2041999999999999e-05 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.816 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258297+0000", 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1349999999999997e-06 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.258303+0000", 2023-10-09T04:47:59.817 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0041628569999999998 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.262466+0000", 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013279399999999999 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.818 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.262599+0000", 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000148131 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.262747+0000", 2023-10-09T04:47:59.819 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0073939330000000001 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.270141+0000", 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0077292380000000003 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.820 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.277870+0000", 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.072906803000000006 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.350777+0000", 2023-10-09T04:47:59.821 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017330700000000001 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.350950+0000", 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000118822 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.822 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:46.351069+0000", 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.092792560999999996 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.823 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:221451 8.9 8:9e118a7f:::benchmark_data_smithi082_175677_object104437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:53.465763+0000", 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 66.174629709000001, 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.075924794000000004, 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.824 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 221451 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465763+0000", 2023-10-09T04:47:59.825 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465763+0000", 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.826 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465762+0000", 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4829999999999996e-06 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465768+0000", 2023-10-09T04:47:59.827 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2037e-05 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465780+0000", 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3610000000000003e-06 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.828 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.465788+0000", 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049860119999999997 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.470774+0000", 2023-10-09T04:47:59.829 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000136954 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.470911+0000", 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000147641 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.830 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.471058+0000", 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0058810110000000002 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.476939+0000", 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013565013000000001 2023-10-09T04:47:59.831 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.490504+0000", 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.051044953999999997 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.541549+0000", 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3082000000000001e-05 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.832 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.541642+0000", 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5309000000000003e-05 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:53.541688+0000", 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.075924794000000004 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.833 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:228739 8.9 8:99c653b5:::benchmark_data_smithi082_175677_object111725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:47:05.481820+0000", 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 54.158572868999997, 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.072494610000000001, 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:47:59.834 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 228739 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481820+0000", 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481820+0000", 2023-10-09T04:47:59.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481817+0000", 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0699e-05 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481828+0000", 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1294999999999999e-05 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481849+0000", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7639999999999994e-06 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481858+0000", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.774e-05 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.481906+0000", 2023-10-09T04:47:59.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011353000000000001 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.482019+0000", 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000113878 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.482133+0000", 2023-10-09T04:47:59.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0059769339999999997 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.488110+0000", 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020597899999999999 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.488316+0000", 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.065842576 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.554159+0000", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000108025 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.554267+0000", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7623999999999997e-05 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:47:05.554314+0000", 2023-10-09T04:47:59.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.072494610000000001 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:47:59.841 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:47:59.841 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:59.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:59.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:59.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:47:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:47:59] "GET /metrics HTTP/1.1" 200 34099 "" "Prometheus/2.33.4" 2023-10-09T04:47:59.996 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:59.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:47:59.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:00.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:00.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:00 smithi153 bash[59565]: cluster 2023-10-09T04:47:58.691946+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1083: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:00.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:00 smithi153 bash[59565]: audit 2023-10-09T04:47:59.288992+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:00.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:00 smithi082 bash[77063]: cluster 2023-10-09T04:47:58.691946+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1083: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:00.238 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:00 smithi082 bash[77063]: audit 2023-10-09T04:47:59.288992+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:00.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:00 smithi082 bash[79733]: cluster 2023-10-09T04:47:58.691946+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1083: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:00.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:00 smithi082 bash[79733]: audit 2023-10-09T04:47:59.288992+0000 mon.a (mon.0) 1496 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:00.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:00.309 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:48:00.340 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:48:00.357 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:00.362 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:48:00.383 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:00.390 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:48:00.415 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:48:00.444 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:48:00.469 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:48:00.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:48:00.496 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:48:00.498 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.499 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:48:00.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:00.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:00.741 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:00.773 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:00.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:00.908 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:00.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:01.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:01.029 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:48:01.029 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_4' created 2023-10-09T04:48:01.034 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:48:01.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:01.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:01.172 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:01.275 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:01.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:01.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.275919+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.82:0/1322417634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:01.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.364331+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.82:0/591295111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.364529+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.558759+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.82:0/1364612324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.724148+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.82:0/2594032327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.847316+0000 mon.a (mon.0) 1498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[77063]: audit 2023-10-09T04:48:00.848763+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.275919+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.82:0/1322417634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.364331+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.82:0/591295111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.364529+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.558759+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.82:0/1364612324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.388 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.724148+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.82:0/2594032327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.388 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.847316+0000 mon.a (mon.0) 1498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.388 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:01 smithi082 bash[79733]: audit 2023-10-09T04:48:00.848763+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:01.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.275919+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.82:0/1322417634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:01.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.364331+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.82:0/591295111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.364529+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:48:01.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.558759+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.82:0/1364612324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.724148+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.82:0/2594032327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.847316+0000 mon.a (mon.0) 1498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:01.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:01 smithi153 bash[59565]: audit 2023-10-09T04:48:00.848763+0000 mon.a (mon.0) 1499 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:01.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:01.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:01.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:01.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:01.709 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:01.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:01.847 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:01.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:01.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:01.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:01.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:02.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:02.111 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:02.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:00.692633+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1084: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:02.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:01.008228+0000 mon.a (mon.0) 1500 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:01.008266+0000 mon.a (mon.0) 1501 : cluster 1 Cluster is now healthy 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:01.017147+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:01.017313+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: audit 2023-10-09T04:48:01.020741+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: audit 2023-10-09T04:48:01.020846+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: audit 2023-10-09T04:48:01.020904+0000 mon.a (mon.0) 1504 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[77063]: cluster 2023-10-09T04:48:01.020971+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:00.692633+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1084: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:01.008228+0000 mon.a (mon.0) 1500 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:02.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:01.008266+0000 mon.a (mon.0) 1501 : cluster 1 Cluster is now healthy 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:01.017147+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:01.017313+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: audit 2023-10-09T04:48:01.020741+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: audit 2023-10-09T04:48:01.020846+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: audit 2023-10-09T04:48:01.020904+0000 mon.a (mon.0) 1504 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:02.161 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:02 smithi082 bash[79733]: cluster 2023-10-09T04:48:01.020971+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:00.692633+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1084: 73 pgs: 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:01.008228+0000 mon.a (mon.0) 1500 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:01.008266+0000 mon.a (mon.0) 1501 : cluster 1 Cluster is now healthy 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:01.017147+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:01.017313+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: audit 2023-10-09T04:48:01.020741+0000 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: audit 2023-10-09T04:48:01.020846+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:02.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: audit 2023-10-09T04:48:01.020904+0000 mon.a (mon.0) 1504 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:02.177 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:02 smithi153 bash[59565]: cluster 2023-10-09T04:48:01.020971+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:48:02.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:02.251 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:02.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:02.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:02.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:02.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:02.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:02.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:02.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:02.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:02.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:02.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:02.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:03.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:03.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:03.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:03.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:03.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.026943+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.156 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.029579+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.030659+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.031408+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.031410+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.026943+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.029579+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.030659+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.031408+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.157 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:03 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.031410+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:03.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:03.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:03.303 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:03.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:03 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.026943+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:03 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.029579+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:48:03.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:03 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.030659+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:03 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.031408+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.305 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:03 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.031410+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T04:48:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:03.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:03.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:03.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:03.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:03.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:03.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.824 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:48:03.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:48:03.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:03.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:48:03.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:03.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:04.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:04.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:04.153 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T04:48:04.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:04.218 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[77063]: cluster 2023-10-09T04:48:02.693019+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1087: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:04.218 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[77063]: cluster 2023-10-09T04:48:03.038091+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.218 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[77063]: cluster 2023-10-09T04:48:03.038327+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.218 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[77063]: cluster 2023-10-09T04:48:03.041704+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:48:04.219 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[79733]: cluster 2023-10-09T04:48:02.693019+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1087: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:04.219 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[79733]: cluster 2023-10-09T04:48:03.038091+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.219 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[79733]: cluster 2023-10-09T04:48:03.038327+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.219 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:04 smithi082 bash[79733]: cluster 2023-10-09T04:48:03.041704+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:48:04.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:04.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:04.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:04 smithi153 bash[59565]: cluster 2023-10-09T04:48:02.693019+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1087: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:04 smithi153 bash[59565]: cluster 2023-10-09T04:48:03.038091+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:04 smithi153 bash[59565]: cluster 2023-10-09T04:48:03.038327+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:04 smithi153 bash[59565]: cluster 2023-10-09T04:48:03.041704+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:48:04.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:04.490 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:04.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:04.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:04.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:04.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:04.759 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:04.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:04.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:04.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:04.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:05.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:05.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:05.134 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:05.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:05.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:05.277 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:48:05.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:05 smithi153 bash[59565]: cluster 2023-10-09T04:48:04.009118+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:05 smithi153 bash[59565]: cluster 2023-10-09T04:48:04.009609+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.311 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:05 smithi082 bash[79733]: cluster 2023-10-09T04:48:04.009118+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.312 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:05 smithi082 bash[79733]: cluster 2023-10-09T04:48:04.009609+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.312 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:05 smithi082 bash[77063]: cluster 2023-10-09T04:48:04.009118+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.312 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:05 smithi082 bash[77063]: cluster 2023-10-09T04:48:04.009609+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T04:48:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:05.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:05.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:05.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:05.583 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:05.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:05.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:05.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:05.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:05.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:05.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:05.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:06.088 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:06.099 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:06 smithi153 bash[59565]: cluster 2023-10-09T04:48:04.693803+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1089: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:06.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:06.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:48:06.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:06.358 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.368 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:06 smithi082 bash[77063]: cluster 2023-10-09T04:48:04.693803+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1089: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:06.369 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:06 smithi082 bash[79733]: cluster 2023-10-09T04:48:04.693803+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1089: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:06.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:06.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:06.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:06.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:06.654 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:06.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:06.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:06.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:06.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:07.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:07.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:07.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:07.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:07.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:07.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:07.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:07.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:07.433 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:07.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:07.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:07.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:07.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:07.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:07.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:07.837 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:07.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:07.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:07.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:08.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:08.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:08.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:08.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:08 smithi153 bash[59565]: cluster 2023-10-09T04:48:06.694863+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:08.114 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:08 smithi153 bash[59565]: audit 2023-10-09T04:48:07.340743+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:08.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:08.236 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:08.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:08.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:08 smithi082 bash[77063]: cluster 2023-10-09T04:48:06.694863+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:08.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:08 smithi082 bash[77063]: audit 2023-10-09T04:48:07.340743+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:08.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:08 smithi082 bash[79733]: cluster 2023-10-09T04:48:06.694863+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:08.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:08 smithi082 bash[79733]: audit 2023-10-09T04:48:07.340743+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:48:08.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:08.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:08.602 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:08.602 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:08.603 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:08.638 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:08.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:08.864 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:48:08.864 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:48:08.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:08.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:09.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:09.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:09.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:09.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:09.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:09.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:09.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:09.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:09.427 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:09.457 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:09.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:09.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:09.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:09.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:09.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:09.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:09.724 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:48:09.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:09.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:09.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:09.956 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:09.969 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:09] "GET /metrics HTTP/1.1" 200 34099 "" "Prometheus/2.33.4" 2023-10-09T04:48:09.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:09.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:10.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:10.115 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:10.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:10 smithi082 bash[77063]: cluster 2023-10-09T04:48:08.695611+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:10.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:10 smithi082 bash[79733]: cluster 2023-10-09T04:48:08.695611+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:10.247 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:10.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:10.363 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:10 smithi153 bash[59565]: cluster 2023-10-09T04:48:08.695611+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:10.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:10.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:10.519 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:10.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:10.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:10.630 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:10.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:10.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:10.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:10.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:11.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:11.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:11.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:11.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:11.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:11.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:11.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:11.454 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:11.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:11.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:11.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:11.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:11.714 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:11.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:11.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:11.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:11.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:11.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:12.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:12.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:12.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:12.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:12.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:12.359 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:12 smithi082 bash[79733]: cluster 2023-10-09T04:48:10.696720+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:12.359 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:12 smithi082 bash[77063]: cluster 2023-10-09T04:48:10.696720+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:12.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:12 smithi153 bash[59565]: cluster 2023-10-09T04:48:10.696720+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:12.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:12.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:12.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:12.629 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:12.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:12.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:12.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:12.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:12.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:12.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:12.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:13.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:13.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:13.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:13.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:13.398 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:13.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:13.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:13.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:13.678 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:13.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:13.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:13.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:14.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:14.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:14.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.201 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:14.261 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:48:14.305 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[77063]: cluster 2023-10-09T04:48:12.697431+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[77063]: audit 2023-10-09T04:48:13.280308+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[77063]: audit 2023-10-09T04:48:13.287004+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[77063]: audit 2023-10-09T04:48:13.631681+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[77063]: audit 2023-10-09T04:48:13.638020+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[79733]: cluster 2023-10-09T04:48:12.697431+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:14.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[79733]: audit 2023-10-09T04:48:13.280308+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[79733]: audit 2023-10-09T04:48:13.287004+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[79733]: audit 2023-10-09T04:48:13.631681+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:14 smithi082 bash[79733]: audit 2023-10-09T04:48:13.638020+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:14.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:14 smithi153 bash[59565]: cluster 2023-10-09T04:48:12.697431+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1093: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:14 smithi153 bash[59565]: audit 2023-10-09T04:48:13.280308+0000 mon.a (mon.0) 1509 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:14 smithi153 bash[59565]: audit 2023-10-09T04:48:13.287004+0000 mon.a (mon.0) 1510 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:14 smithi153 bash[59565]: audit 2023-10-09T04:48:13.631681+0000 mon.a (mon.0) 1511 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:14 smithi153 bash[59565]: audit 2023-10-09T04:48:13.638020+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:14.430 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:14.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:14.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:14.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:14.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:14.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:14.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.719 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:14.822 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:14.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:14.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:14.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:15.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:15.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.218099+0000 mon.a (mon.0) 1513 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:15.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.219063+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:15.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.225167+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.096 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.292876+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.293863+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:15.097 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:15 smithi153 bash[59565]: audit 2023-10-09T04:48:14.901579+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:15.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:15.217 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:15.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:15.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:15.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.218099+0000 mon.a (mon.0) 1513 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.219063+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.225167+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.292876+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.293863+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:15.415 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[79733]: audit 2023-10-09T04:48:14.901579+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.218099+0000 mon.a (mon.0) 1513 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.219063+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.225167+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.292876+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.293863+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:15.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:15 smithi082 bash[77063]: audit 2023-10-09T04:48:14.901579+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:15.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:15.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:15.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:15.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:15.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:15.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:15.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:15.906 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:15.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:16.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:16.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:16.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:16.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:16 smithi153 bash[59565]: cluster 2023-10-09T04:48:14.698262+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:16.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:16.339 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:16 smithi082 bash[77063]: cluster 2023-10-09T04:48:14.698262+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:16.339 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:16 smithi082 bash[79733]: cluster 2023-10-09T04:48:14.698262+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1094: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:16.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:16.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:16.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:16.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:16.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:16.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:16.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:16.801 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:16.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:16.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:16.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:16.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:17.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:17.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:17.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:17.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:17.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:17.341 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:17.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:17.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:17.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:17.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:17.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:17.608 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:17.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:17.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:17.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:17.752 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:48:17.753 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:48:17.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:17.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:17.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:17.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:17.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:18.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:18.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:18.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:18.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:18.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:18.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:18 smithi153 bash[59565]: cluster 2023-10-09T04:48:16.698953+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:18.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:18.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:18.415 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:18.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:18 smithi082 bash[77063]: cluster 2023-10-09T04:48:16.698953+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:18.460 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:18 smithi082 bash[79733]: cluster 2023-10-09T04:48:16.698953+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:18.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:18.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:18.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:48:18.653 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:18.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:18.829 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:18.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:18.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:18.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:18.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:18.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:19.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:19.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:19.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:19.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:19.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:19.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:19.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:19.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:19.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:19.582 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:48:19.609 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:48:19.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.630 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:48:19.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:19.651 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:48:19.674 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:48:19.703 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:48:19.728 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:48:19.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.752 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:48:19.753 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.753 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.753 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.754 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.754 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.754 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.754 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.754 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:48:19.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:19.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:19.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:19.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:19.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:19.954 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:19] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-09T04:48:20.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:20.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:20.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:20 smithi153 bash[59565]: cluster 2023-10-09T04:48:18.699567+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:20.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:20 smithi153 bash[59565]: audit 2023-10-09T04:48:19.558899+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.82:0/860368075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:20.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:20.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:20 smithi082 bash[77063]: cluster 2023-10-09T04:48:18.699567+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:20.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:20 smithi082 bash[77063]: audit 2023-10-09T04:48:19.558899+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.82:0/860368075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:20 smithi082 bash[79733]: cluster 2023-10-09T04:48:18.699567+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:20 smithi082 bash[79733]: audit 2023-10-09T04:48:19.558899+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.82:0/860368075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:20.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:20.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:20.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:20.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:20.593 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:20.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:20.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:20.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:20.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:20.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:20.987 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:20.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:21.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:21.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:21.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:21.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:21.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:21.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:21.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:21.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:21.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:21.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:21.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:21.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:21.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:21.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:21.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:21.883 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:21.914 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:21.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:22.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:22.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:22.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:22 smithi153 bash[59565]: cluster 2023-10-09T04:48:20.700175+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:22.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:22.283 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:22.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:22.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:22.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:22.422 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:22 smithi082 bash[77063]: cluster 2023-10-09T04:48:20.700175+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:22.422 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:22 smithi082 bash[79733]: cluster 2023-10-09T04:48:20.700175+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:22.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:22.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:22.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:22.681 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:22.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:22.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:22.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:22.952 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:22.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:22.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:23.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:23.121 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:23.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:23.252 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:23.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:23.381 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:23.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:23.429 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:48:23.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:23.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:23.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:23.648 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:48:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.650 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:23.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:23.651 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:48:23.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:23.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:23.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:23.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:23.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:23.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:24.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:24.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:24.109 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:48:24.109 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T04:48:24.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:24.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:24.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:24.295 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:24.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:24.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:24.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:24 smithi153 bash[59565]: cluster 2023-10-09T04:48:22.700779+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:24.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:24 smithi153 bash[59565]: audit 2023-10-09T04:48:23.577394+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.82:0/618884547' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:24 smithi153 bash[59565]: audit 2023-10-09T04:48:23.578107+0000 mon.a (mon.0) 1519 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:24 smithi153 bash[59565]: audit 2023-10-09T04:48:23.795598+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.82:0/1185503971' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:24 smithi153 bash[59565]: audit 2023-10-09T04:48:23.796163+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[77063]: cluster 2023-10-09T04:48:22.700779+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[77063]: audit 2023-10-09T04:48:23.577394+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.82:0/618884547' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[77063]: audit 2023-10-09T04:48:23.578107+0000 mon.a (mon.0) 1519 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[77063]: audit 2023-10-09T04:48:23.795598+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.82:0/1185503971' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[77063]: audit 2023-10-09T04:48:23.796163+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[79733]: cluster 2023-10-09T04:48:22.700779+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[79733]: audit 2023-10-09T04:48:23.577394+0000 mon.c (mon.2) 113 : audit 1 from='client.? 172.21.15.82:0/618884547' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[79733]: audit 2023-10-09T04:48:23.578107+0000 mon.a (mon.0) 1519 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[79733]: audit 2023-10-09T04:48:23.795598+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.82:0/1185503971' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:24 smithi082 bash[79733]: audit 2023-10-09T04:48:23.796163+0000 mon.a (mon.0) 1520 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:24.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:24.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:24.657 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:24.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:24.785 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:24.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:24.876 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1098,"stamp":"2023-10-09T04:48:22.700356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8398292,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3891936,"kb_used_data":34016,"kb_used_omap":47,"kb_used_meta":3857808,"kb_avail":746069280,"statfs":{"total":767960285184,"available":763974942720,"internally_reserved":0,"allocated":34832384,"data_stored":28521024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":3950395805},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004463"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":248,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040036+0000","last_change":"2023-10-09T04:44:51.896564+0000","last_active":"2023-10-09T04:48:02.040036+0000","last_peered":"2023-10-09T04:48:02.040036+0000","last_clean":"2023-10-09T04:48:02.040036+0000","last_became_active":"2023-10-09T04:44:51.896195+0000","last_became_peered":"2023-10-09T04:44:51.896195+0000","last_unstale":"2023-10-09T04:48:02.040036+0000","last_undegraded":"2023-10-09T04:48:02.040036+0000","last_fullsized":"2023-10-09T04:48:02.040036+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.029273+0000","last_change":"2023-10-09T04:44:51.891799+0000","last_active":"2023-10-09T04:48:02.029273+0000","last_peered":"2023-10-09T04:48:02.029273+0000","last_clean":"2023-10-09T04:48:02.029273+0000","last_became_active":"2023-10-09T04:44:51.891479+0000","last_became_peered":"2023-10-09T04:44:51.891479+0000","last_unstale":"2023-10-09T04:48:02.029273+0000","last_undegraded":"2023-10-09T04:48:02.029273+0000","last_fullsized":"2023-10-09T04:48:02.029273+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:57.730414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":256,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031029+0000","last_change":"2023-10-09T04:43:53.989477+0000","last_active":"2023-10-09T04:48:02.031029+0000","last_peered":"2023-10-09T04:48:02.031029+0000","last_clean":"2023-10-09T04:48:02.031029+0000","last_became_active":"2023-10-09T04:43:53.988910+0000","last_became_peered":"2023-10-09T04:43:53.988910+0000","last_unstale":"2023-10-09T04:48:02.031029+0000","last_undegraded":"2023-10-09T04:48:02.031029+0000","last_fullsized":"2023-10-09T04:48:02.031029+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:35.144801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":229,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.034072+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:48:02.034072+0000","last_peered":"2023-10-09T04:48:02.034072+0000","last_clean":"2023-10-09T04:48:02.034072+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:48:02.034072+0000","last_undegraded":"2023-10-09T04:48:02.034072+0000","last_fullsized":"2023-10-09T04:48:02.034072+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":233,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.038427+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:48:03.038427+0000","last_peered":"2023-10-09T04:48:03.038427+0000","last_clean":"2023-10-09T04:48:03.038427+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:48:03.038427+0000","last_undegraded":"2023-10-09T04:48:03.038427+0000","last_fullsized":"2023-10-09T04:48:03.038427+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.036671+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:48:02.036671+0000","last_peered":"2023-10-09T04:48:02.036671+0000","last_clean":"2023-10-09T04:48:02.036671+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:48:02.036671+0000","last_undegraded":"2023-10-09T04:48:02.036671+0000","last_fullsized":"2023-10-09T04:48:02.036671+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033605+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:48:02.033605+0000","last_peered":"2023-10-09T04:48:02.033605+0000","last_clean":"2023-10-09T04:48:02.033605+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:48:02.033605+0000","last_undegraded":"2023-10-09T04:48:02.033605+0000","last_fullsized":"2023-10-09T04:48:02.033605+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.154635+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:48:02.154635+0000","last_peered":"2023-10-09T04:48:02.154635+0000","last_clean":"2023-10-09T04:48:02.154635+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:48:02.154635+0000","last_undegraded":"2023-10-09T04:48:02.154635+0000","last_fullsized":"2023-10-09T04:48:02.154635+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":193,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.043386+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:48:02.043386+0000","last_peered":"2023-10-09T04:48:02.043386+0000","last_clean":"2023-10-09T04:48:02.043386+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:48:02.043386+0000","last_undegraded":"2023-10-09T04:48:02.043386+0000","last_fullsized":"2023-10-09T04:48:02.043386+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038336+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:48:02.038336+0000","last_peered":"2023-10-09T04:48:02.038336+0000","last_clean":"2023-10-09T04:48:02.038336+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:48:02.038336+0000","last_undegraded":"2023-10-09T04:48:02.038336+0000","last_fullsized":"2023-10-09T04:48:02.038336+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042387+0000","last_change":"2023-10-09T04:40:47.996370+0000","last_active":"2023-10-09T04:48:02.042387+0000","last_peered":"2023-10-09T04:48:02.042387+0000","last_clean":"2023-10-09T04:48:02.042387+0000","last_became_active":"2023-10-09T04:40:47.817084+0000","last_became_peered":"2023-10-09T04:40:47.817084+0000","last_unstale":"2023-10-09T04:48:02.042387+0000","last_undegraded":"2023-10-09T04:48:02.042387+0000","last_fullsized":"2023-10-09T04:48:02.042387+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":235,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033143+0000","last_change":"2023-10-09T04:44:51.903464+0000","last_active":"2023-10-09T04:48:02.033143+0000","last_peered":"2023-10-09T04:48:02.033143+0000","last_clean":"2023-10-09T04:48:02.033143+0000","last_became_active":"2023-10-09T04:44:51.903078+0000","last_became_peered":"2023-10-09T04:44:51.903078+0000","last_unstale":"2023-10-09T04:48:02.033143+0000","last_undegraded":"2023-10-09T04:48:02.033143+0000","last_fullsized":"2023-10-09T04:48:02.033143+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:57.992171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":277,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040905+0000","last_change":"2023-10-09T04:40:47.863039+0000","last_active":"2023-10-09T04:48:02.040905+0000","last_peered":"2023-10-09T04:48:02.040905+0000","last_clean":"2023-10-09T04:48:02.040905+0000","last_became_active":"2023-10-09T04:40:47.819279+0000","last_became_peered":"2023-10-09T04:40:47.819279+0000","last_unstale":"2023-10-09T04:48:02.040905+0000","last_undegraded":"2023-10-09T04:48:02.040905+0000","last_fullsized":"2023-10-09T04:48:02.040905+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040523+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:48:02.040523+0000","last_peered":"2023-10-09T04:48:02.040523+0000","last_clean":"2023-10-09T04:48:02.040523+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:48:02.040523+0000","last_undegraded":"2023-10-09T04:48:02.040523+0000","last_fullsized":"2023-10-09T04:48:02.040523+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":88,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038908+0000","last_change":"2023-10-09T04:44:51.899516+0000","last_active":"2023-10-09T04:48:02.038908+0000","last_peered":"2023-10-09T04:48:02.038908+0000","last_clean":"2023-10-09T04:48:02.038908+0000","last_became_active":"2023-10-09T04:44:51.899092+0000","last_became_peered":"2023-10-09T04:44:51.899092+0000","last_unstale":"2023-10-09T04:48:02.038908+0000","last_undegraded":"2023-10-09T04:48:02.038908+0000","last_fullsized":"2023-10-09T04:48:02.038908+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031768+0000","last_change":"2023-10-09T04:40:47.816775+0000","last_active":"2023-10-09T04:48:02.031768+0000","last_peered":"2023-10-09T04:48:02.031768+0000","last_clean":"2023-10-09T04:48:02.031768+0000","last_became_active":"2023-10-09T04:40:47.816424+0000","last_became_peered":"2023-10-09T04:40:47.816424+0000","last_unstale":"2023-10-09T04:48:02.031768+0000","last_undegraded":"2023-10-09T04:48:02.031768+0000","last_fullsized":"2023-10-09T04:48:02.031768+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:52.923977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":228,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.038632+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:48:03.038632+0000","last_peered":"2023-10-09T04:48:03.038632+0000","last_clean":"2023-10-09T04:48:03.038632+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:48:03.038632+0000","last_undegraded":"2023-10-09T04:48:03.038632+0000","last_fullsized":"2023-10-09T04:48:03.038632+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":202,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042198+0000","last_change":"2023-10-09T04:40:47.820893+0000","last_active":"2023-10-09T04:48:02.042198+0000","last_peered":"2023-10-09T04:48:02.042198+0000","last_clean":"2023-10-09T04:48:02.042198+0000","last_became_active":"2023-10-09T04:40:47.820570+0000","last_became_peered":"2023-10-09T04:40:47.820570+0000","last_unstale":"2023-10-09T04:48:02.042198+0000","last_undegraded":"2023-10-09T04:48:02.042198+0000","last_fullsized":"2023-10-09T04:48:02.042198+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":67,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038264+0000","last_change":"2023-10-09T04:40:47.823962+0000","last_active":"2023-10-09T04:48:02.038264+0000","last_peered":"2023-10-09T04:48:02.038264+0000","last_clean":"2023-10-09T04:48:02.038264+0000","last_became_active":"2023-10-09T04:40:47.823509+0000","last_became_peered":"2023-10-09T04:40:47.823509+0000","last_unstale":"2023-10-09T04:48:02.038264+0000","last_undegraded":"2023-10-09T04:48:02.038264+0000","last_fullsized":"2023-10-09T04:48:02.038264+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":227,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033975+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:48:02.033975+0000","last_peered":"2023-10-09T04:48:02.033975+0000","last_clean":"2023-10-09T04:48:02.033975+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:48:02.033975+0000","last_undegraded":"2023-10-09T04:48:02.033975+0000","last_fullsized":"2023-10-09T04:48:02.033975+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":216,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.035394+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:48:02.035394+0000","last_peered":"2023-10-09T04:48:02.035394+0000","last_clean":"2023-10-09T04:48:02.035394+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:48:02.035394+0000","last_undegraded":"2023-10-09T04:48:02.035394+0000","last_fullsized":"2023-10-09T04:48:02.035394+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":145,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.155872+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:48:02.155872+0000","last_peered":"2023-10-09T04:48:02.155872+0000","last_clean":"2023-10-09T04:48:02.155872+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:48:02.155872+0000","last_undegraded":"2023-10-09T04:48:02.155872+0000","last_fullsized":"2023-10-09T04:48:02.155872+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039380+0000","last_change":"2023-10-09T04:48:02.044407+0000","last_active":"2023-10-09T04:48:03.039380+0000","last_peered":"2023-10-09T04:48:03.039380+0000","last_clean":"2023-10-09T04:48:03.039380+0000","last_became_active":"2023-10-09T04:48:02.043959+0000","last_became_peered":"2023-10-09T04:48:02.043959+0000","last_unstale":"2023-10-09T04:48:03.039380+0000","last_undegraded":"2023-10-09T04:48:03.039380+0000","last_fullsized":"2023-10-09T04:48:03.039380+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:02.677050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038399+0000","last_change":"2023-10-09T04:40:48.230655+0000","last_active":"2023-10-09T04:48:02.038399+0000","last_peered":"2023-10-09T04:48:02.038399+0000","last_clean":"2023-10-09T04:48:02.038399+0000","last_became_active":"2023-10-09T04:40:47.830826+0000","last_became_peered":"2023-10-09T04:40:47.830826+0000","last_unstale":"2023-10-09T04:48:02.038399+0000","last_undegraded":"2023-10-09T04:48:02.038399+0000","last_fullsized":"2023-10-09T04:48:02.038399+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":184,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040766+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:48:02.040766+0000","last_peered":"2023-10-09T04:48:02.040766+0000","last_clean":"2023-10-09T04:48:02.040766+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:48:02.040766+0000","last_undegraded":"2023-10-09T04:48:02.040766+0000","last_fullsized":"2023-10-09T04:48:02.040766+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442858+0000","last_change":"2023-10-09T04:48:02.139233+0000","last_active":"2023-10-09T04:48:03.442858+0000","last_peered":"2023-10-09T04:48:03.442858+0000","last_clean":"2023-10-09T04:48:03.442858+0000","last_became_active":"2023-10-09T04:48:02.138632+0000","last_became_peered":"2023-10-09T04:48:02.138632+0000","last_unstale":"2023-10-09T04:48:03.442858+0000","last_undegraded":"2023-10-09T04:48:03.442858+0000","last_fullsized":"2023-10-09T04:48:03.442858+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:52.944610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":84,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038496+0000","last_change":"2023-10-09T04:43:54.055892+0000","last_active":"2023-10-09T04:48:02.038496+0000","last_peered":"2023-10-09T04:48:02.038496+0000","last_clean":"2023-10-09T04:48:02.038496+0000","last_became_active":"2023-10-09T04:43:53.987210+0000","last_became_peered":"2023-10-09T04:43:53.987210+0000","last_unstale":"2023-10-09T04:48:02.038496+0000","last_undegraded":"2023-10-09T04:48:02.038496+0000","last_fullsized":"2023-10-09T04:48:02.038496+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040873+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:48:02.040873+0000","last_peered":"2023-10-09T04:48:02.040873+0000","last_clean":"2023-10-09T04:48:02.040873+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:48:02.040873+0000","last_undegraded":"2023-10-09T04:48:02.040873+0000","last_fullsized":"2023-10-09T04:48:02.040873+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.158330+0000","last_change":"2023-10-09T04:48:02.157308+0000","last_active":"2023-10-09T04:48:02.158330+0000","last_peered":"2023-10-09T04:48:02.158330+0000","last_clean":"2023-10-09T04:48:02.158330+0000","last_became_active":"2023-10-09T04:48:02.156513+0000","last_became_peered":"2023-10-09T04:48:02.156513+0000","last_unstale":"2023-10-09T04:48:02.158330+0000","last_undegraded":"2023-10-09T04:48:02.158330+0000","last_fullsized":"2023-10-09T04:48:02.158330+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":71,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042529+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:48:02.042529+0000","last_peered":"2023-10-09T04:48:02.042529+0000","last_clean":"2023-10-09T04:48:02.042529+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:48:02.042529+0000","last_undegraded":"2023-10-09T04:48:02.042529+0000","last_fullsized":"2023-10-09T04:48:02.042529+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.035545+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:48:02.035545+0000","last_peered":"2023-10-09T04:48:02.035545+0000","last_clean":"2023-10-09T04:48:02.035545+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:48:02.035545+0000","last_undegraded":"2023-10-09T04:48:02.035545+0000","last_fullsized":"2023-10-09T04:48:02.035545+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.045524+0000","last_change":"2023-10-09T04:48:02.045217+0000","last_active":"2023-10-09T04:48:02.045524+0000","last_peered":"2023-10-09T04:48:02.045524+0000","last_clean":"2023-10-09T04:48:02.045524+0000","last_became_active":"2023-10-09T04:48:02.044927+0000","last_became_peered":"2023-10-09T04:48:02.044927+0000","last_unstale":"2023-10-09T04:48:02.045524+0000","last_undegraded":"2023-10-09T04:48:02.045524+0000","last_fullsized":"2023-10-09T04:48:02.045524+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":185,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041859+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:48:02.041859+0000","last_peered":"2023-10-09T04:48:02.041859+0000","last_clean":"2023-10-09T04:48:02.041859+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:48:02.041859+0000","last_undegraded":"2023-10-09T04:48:02.041859+0000","last_fullsized":"2023-10-09T04:48:02.041859+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":238,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031719+0000","last_change":"2023-10-09T04:44:51.892263+0000","last_active":"2023-10-09T04:48:02.031719+0000","last_peered":"2023-10-09T04:48:02.031719+0000","last_clean":"2023-10-09T04:48:02.031719+0000","last_became_active":"2023-10-09T04:44:51.891917+0000","last_became_peered":"2023-10-09T04:44:51.891917+0000","last_unstale":"2023-10-09T04:48:02.031719+0000","last_undegraded":"2023-10-09T04:48:02.031719+0000","last_fullsized":"2023-10-09T04:48:02.031719+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.796713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.157889+0000","last_change":"2023-10-09T04:48:02.156998+0000","last_active":"2023-10-09T04:48:02.157889+0000","last_peered":"2023-10-09T04:48:02.157889+0000","last_clean":"2023-10-09T04:48:02.157889+0000","last_became_active":"2023-10-09T04:48:02.156119+0000","last_became_peered":"2023-10-09T04:48:02.156119+0000","last_unstale":"2023-10-09T04:48:02.157889+0000","last_undegraded":"2023-10-09T04:48:02.157889+0000","last_fullsized":"2023-10-09T04:48:02.157889+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":173,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033209+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:48:02.033209+0000","last_peered":"2023-10-09T04:48:02.033209+0000","last_clean":"2023-10-09T04:48:02.033209+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:48:02.033209+0000","last_undegraded":"2023-10-09T04:48:02.033209+0000","last_fullsized":"2023-10-09T04:48:02.033209+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041053+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:48:02.041053+0000","last_peered":"2023-10-09T04:48:02.041053+0000","last_clean":"2023-10-09T04:48:02.041053+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:48:02.041053+0000","last_undegraded":"2023-10-09T04:48:02.041053+0000","last_fullsized":"2023-10-09T04:48:02.041053+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442310+0000","last_change":"2023-10-09T04:48:02.036963+0000","last_active":"2023-10-09T04:48:03.442310+0000","last_peered":"2023-10-09T04:48:03.442310+0000","last_clean":"2023-10-09T04:48:03.442310+0000","last_became_active":"2023-10-09T04:48:02.036549+0000","last_became_peered":"2023-10-09T04:48:02.036549+0000","last_unstale":"2023-10-09T04:48:03.442310+0000","last_undegraded":"2023-10-09T04:48:03.442310+0000","last_fullsized":"2023-10-09T04:48:03.442310+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.195147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.035400+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:48:02.035400+0000","last_peered":"2023-10-09T04:48:02.035400+0000","last_clean":"2023-10-09T04:48:02.035400+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:48:02.035400+0000","last_undegraded":"2023-10-09T04:48:02.035400+0000","last_fullsized":"2023-10-09T04:48:02.035400+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040446+0000","last_change":"2023-10-09T04:44:53.333649+0000","last_active":"2023-10-09T04:48:02.040446+0000","last_peered":"2023-10-09T04:48:02.040446+0000","last_clean":"2023-10-09T04:48:02.040446+0000","last_became_active":"2023-10-09T04:44:53.332464+0000","last_became_peered":"2023-10-09T04:44:53.332464+0000","last_unstale":"2023-10-09T04:48:02.040446+0000","last_undegraded":"2023-10-09T04:48:02.040446+0000","last_fullsized":"2023-10-09T04:48:02.040446+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.140930+0000","last_change":"2023-10-09T04:48:02.140445+0000","last_active":"2023-10-09T04:48:02.140930+0000","last_peered":"2023-10-09T04:48:02.140930+0000","last_clean":"2023-10-09T04:48:02.140930+0000","last_became_active":"2023-10-09T04:48:02.139406+0000","last_became_peered":"2023-10-09T04:48:02.139406+0000","last_unstale":"2023-10-09T04:48:02.140930+0000","last_undegraded":"2023-10-09T04:48:02.140930+0000","last_fullsized":"2023-10-09T04:48:02.140930+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040426+0000","last_change":"2023-10-09T04:44:53.102191+0000","last_active":"2023-10-09T04:48:02.040426+0000","last_peered":"2023-10-09T04:48:02.040426+0000","last_clean":"2023-10-09T04:48:02.040426+0000","last_became_active":"2023-10-09T04:44:53.101206+0000","last_became_peered":"2023-10-09T04:44:53.101206+0000","last_unstale":"2023-10-09T04:48:02.040426+0000","last_undegraded":"2023-10-09T04:48:02.040426+0000","last_fullsized":"2023-10-09T04:48:02.040426+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041175+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:48:02.041175+0000","last_peered":"2023-10-09T04:48:02.041175+0000","last_clean":"2023-10-09T04:48:02.041175+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:48:02.041175+0000","last_undegraded":"2023-10-09T04:48:02.041175+0000","last_fullsized":"2023-10-09T04:48:02.041175+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442634+0000","last_change":"2023-10-09T04:48:02.139290+0000","last_active":"2023-10-09T04:48:03.442634+0000","last_peered":"2023-10-09T04:48:03.442634+0000","last_clean":"2023-10-09T04:48:03.442634+0000","last_became_active":"2023-10-09T04:48:02.138631+0000","last_became_peered":"2023-10-09T04:48:02.138631+0000","last_unstale":"2023-10-09T04:48:03.442634+0000","last_undegraded":"2023-10-09T04:48:03.442634+0000","last_fullsized":"2023-10-09T04:48:03.442634+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:32.990209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.154264+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:48:02.154264+0000","last_peered":"2023-10-09T04:48:02.154264+0000","last_clean":"2023-10-09T04:48:02.154264+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:48:02.154264+0000","last_undegraded":"2023-10-09T04:48:02.154264+0000","last_fullsized":"2023-10-09T04:48:02.154264+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031818+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:48:02.031818+0000","last_peered":"2023-10-09T04:48:02.031818+0000","last_clean":"2023-10-09T04:48:02.031818+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:48:02.031818+0000","last_undegraded":"2023-10-09T04:48:02.031818+0000","last_fullsized":"2023-10-09T04:48:02.031818+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.158297+0000","last_change":"2023-10-09T04:48:02.156639+0000","last_active":"2023-10-09T04:48:02.158297+0000","last_peered":"2023-10-09T04:48:02.158297+0000","last_clean":"2023-10-09T04:48:02.158297+0000","last_became_active":"2023-10-09T04:48:02.155893+0000","last_became_peered":"2023-10-09T04:48:02.155893+0000","last_unstale":"2023-10-09T04:48:02.158297+0000","last_undegraded":"2023-10-09T04:48:02.158297+0000","last_fullsized":"2023-10-09T04:48:02.158297+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":706,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.032081+0000","last_change":"2023-10-09T04:41:04.902111+0000","last_active":"2023-10-09T04:48:02.032081+0000","last_peered":"2023-10-09T04:48:02.032081+0000","last_clean":"2023-10-09T04:48:02.032081+0000","last_became_active":"2023-10-09T04:41:04.901604+0000","last_became_peered":"2023-10-09T04:41:04.901604+0000","last_unstale":"2023-10-09T04:48:02.032081+0000","last_undegraded":"2023-10-09T04:48:02.032081+0000","last_fullsized":"2023-10-09T04:48:02.032081+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:55.374183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":215,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042294+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:48:02.042294+0000","last_peered":"2023-10-09T04:48:02.042294+0000","last_clean":"2023-10-09T04:48:02.042294+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:48:02.042294+0000","last_undegraded":"2023-10-09T04:48:02.042294+0000","last_fullsized":"2023-10-09T04:48:02.042294+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":225,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033268+0000","last_change":"2023-10-09T04:43:53.993014+0000","last_active":"2023-10-09T04:48:02.033268+0000","last_peered":"2023-10-09T04:48:02.033268+0000","last_clean":"2023-10-09T04:48:02.033268+0000","last_became_active":"2023-10-09T04:43:53.992702+0000","last_became_peered":"2023-10-09T04:43:53.992702+0000","last_unstale":"2023-10-09T04:48:02.033268+0000","last_undegraded":"2023-10-09T04:48:02.033268+0000","last_fullsized":"2023-10-09T04:48:02.033268+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:47.151693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.038544+0000","last_change":"2023-10-09T04:48:02.038776+0000","last_active":"2023-10-09T04:48:03.038544+0000","last_peered":"2023-10-09T04:48:03.038544+0000","last_clean":"2023-10-09T04:48:03.038544+0000","last_became_active":"2023-10-09T04:48:02.038418+0000","last_became_peered":"2023-10-09T04:48:02.038418+0000","last_unstale":"2023-10-09T04:48:03.038544+0000","last_undegraded":"2023-10-09T04:48:03.038544+0000","last_fullsized":"2023-10-09T04:48:03.038544+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:26.911517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":714,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041656+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:48:02.041656+0000","last_peered":"2023-10-09T04:48:02.041656+0000","last_clean":"2023-10-09T04:48:02.041656+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:48:02.041656+0000","last_undegraded":"2023-10-09T04:48:02.041656+0000","last_fullsized":"2023-10-09T04:48:02.041656+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":232,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040183+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:48:02.040183+0000","last_peered":"2023-10-09T04:48:02.040183+0000","last_clean":"2023-10-09T04:48:02.040183+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:48:02.040183+0000","last_undegraded":"2023-10-09T04:48:02.040183+0000","last_fullsized":"2023-10-09T04:48:02.040183+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031887+0000","last_change":"2023-10-09T04:44:51.896804+0000","last_active":"2023-10-09T04:48:02.031887+0000","last_peered":"2023-10-09T04:48:02.031887+0000","last_clean":"2023-10-09T04:48:02.031887+0000","last_became_active":"2023-10-09T04:44:51.896434+0000","last_became_peered":"2023-10-09T04:44:51.896434+0000","last_unstale":"2023-10-09T04:48:02.031887+0000","last_undegraded":"2023-10-09T04:48:02.031887+0000","last_fullsized":"2023-10-09T04:48:02.031887+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:39.726149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.046707+0000","last_change":"2023-10-09T04:48:02.046283+0000","last_active":"2023-10-09T04:48:02.046707+0000","last_peered":"2023-10-09T04:48:02.046707+0000","last_clean":"2023-10-09T04:48:02.046707+0000","last_became_active":"2023-10-09T04:48:02.045795+0000","last_became_peered":"2023-10-09T04:48:02.045795+0000","last_unstale":"2023-10-09T04:48:02.046707+0000","last_undegraded":"2023-10-09T04:48:02.046707+0000","last_fullsized":"2023-10-09T04:48:02.046707+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1329,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.028327+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:48:02.028327+0000","last_peered":"2023-10-09T04:48:02.028327+0000","last_clean":"2023-10-09T04:48:02.028327+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:48:02.028327+0000","last_undegraded":"2023-10-09T04:48:02.028327+0000","last_fullsized":"2023-10-09T04:48:02.028327+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.032455+0000","last_change":"2023-10-09T04:44:51.903679+0000","last_active":"2023-10-09T04:48:02.032455+0000","last_peered":"2023-10-09T04:48:02.032455+0000","last_clean":"2023-10-09T04:48:02.032455+0000","last_became_active":"2023-10-09T04:44:51.903243+0000","last_became_peered":"2023-10-09T04:44:51.903243+0000","last_unstale":"2023-10-09T04:48:02.032455+0000","last_undegraded":"2023-10-09T04:48:02.032455+0000","last_fullsized":"2023-10-09T04:48:02.032455+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:35.333537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":251,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031900+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:48:02.031900+0000","last_peered":"2023-10-09T04:48:02.031900+0000","last_clean":"2023-10-09T04:48:02.031900+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:48:02.031900+0000","last_undegraded":"2023-10-09T04:48:02.031900+0000","last_fullsized":"2023-10-09T04:48:02.031900+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039879+0000","last_change":"2023-10-09T04:48:02.139821+0000","last_active":"2023-10-09T04:48:03.039879+0000","last_peered":"2023-10-09T04:48:03.039879+0000","last_clean":"2023-10-09T04:48:03.039879+0000","last_became_active":"2023-10-09T04:48:02.139065+0000","last_became_peered":"2023-10-09T04:48:02.139065+0000","last_unstale":"2023-10-09T04:48:03.039879+0000","last_undegraded":"2023-10-09T04:48:03.039879+0000","last_fullsized":"2023-10-09T04:48:03.039879+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:29.788840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":777,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041821+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:48:02.041821+0000","last_peered":"2023-10-09T04:48:02.041821+0000","last_clean":"2023-10-09T04:48:02.041821+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:48:02.041821+0000","last_undegraded":"2023-10-09T04:48:02.041821+0000","last_fullsized":"2023-10-09T04:48:02.041821+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":473,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038219+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:48:02.038219+0000","last_peered":"2023-10-09T04:48:02.038219+0000","last_clean":"2023-10-09T04:48:02.038219+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:48:02.038219+0000","last_undegraded":"2023-10-09T04:48:02.038219+0000","last_fullsized":"2023-10-09T04:48:02.038219+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.041550+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:48:02.041550+0000","last_peered":"2023-10-09T04:48:02.041550+0000","last_clean":"2023-10-09T04:48:02.041550+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:48:02.041550+0000","last_undegraded":"2023-10-09T04:48:02.041550+0000","last_fullsized":"2023-10-09T04:48:02.041550+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":186,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.036475+0000","last_change":"2023-10-09T04:40:47.825563+0000","last_active":"2023-10-09T04:48:02.036475+0000","last_peered":"2023-10-09T04:48:02.036475+0000","last_clean":"2023-10-09T04:48:02.036475+0000","last_became_active":"2023-10-09T04:40:47.824020+0000","last_became_peered":"2023-10-09T04:40:47.824020+0000","last_unstale":"2023-10-09T04:48:02.036475+0000","last_undegraded":"2023-10-09T04:48:02.036475+0000","last_fullsized":"2023-10-09T04:48:02.036475+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":11,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.038954+0000","last_change":"2023-10-09T04:48:02.042880+0000","last_active":"2023-10-09T04:48:03.038954+0000","last_peered":"2023-10-09T04:48:03.038954+0000","last_clean":"2023-10-09T04:48:03.038954+0000","last_became_active":"2023-10-09T04:48:02.042529+0000","last_became_peered":"2023-10-09T04:48:02.042529+0000","last_unstale":"2023-10-09T04:48:03.038954+0000","last_undegraded":"2023-10-09T04:48:03.038954+0000","last_fullsized":"2023-10-09T04:48:03.038954+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:32.083166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":639,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.037937+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:48:02.037937+0000","last_peered":"2023-10-09T04:48:02.037937+0000","last_clean":"2023-10-09T04:48:02.037937+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:48:02.037937+0000","last_undegraded":"2023-10-09T04:48:02.037937+0000","last_fullsized":"2023-10-09T04:48:02.037937+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":258,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031850+0000","last_change":"2023-10-09T04:43:53.972134+0000","last_active":"2023-10-09T04:48:02.031850+0000","last_peered":"2023-10-09T04:48:02.031850+0000","last_clean":"2023-10-09T04:48:02.031850+0000","last_became_active":"2023-10-09T04:43:53.971710+0000","last_became_peered":"2023-10-09T04:43:53.971710+0000","last_unstale":"2023-10-09T04:48:02.031850+0000","last_undegraded":"2023-10-09T04:48:02.031850+0000","last_fullsized":"2023-10-09T04:48:02.031850+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:56.047480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":240,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033708+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:48:02.033708+0000","last_peered":"2023-10-09T04:48:02.033708+0000","last_clean":"2023-10-09T04:48:02.033708+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:48:02.033708+0000","last_undegraded":"2023-10-09T04:48:02.033708+0000","last_fullsized":"2023-10-09T04:48:02.033708+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.043662+0000","last_change":"2023-10-09T04:48:02.043240+0000","last_active":"2023-10-09T04:48:02.043662+0000","last_peered":"2023-10-09T04:48:02.043662+0000","last_clean":"2023-10-09T04:48:02.043662+0000","last_became_active":"2023-10-09T04:48:02.042897+0000","last_became_peered":"2023-10-09T04:48:02.042897+0000","last_unstale":"2023-10-09T04:48:02.043662+0000","last_undegraded":"2023-10-09T04:48:02.043662+0000","last_fullsized":"2023-10-09T04:48:02.043662+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":469,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042520+0000","last_change":"2023-10-09T04:40:48.818879+0000","last_active":"2023-10-09T04:48:02.042520+0000","last_peered":"2023-10-09T04:48:02.042520+0000","last_clean":"2023-10-09T04:48:02.042520+0000","last_became_active":"2023-10-09T04:40:48.818073+0000","last_became_peered":"2023-10-09T04:40:48.818073+0000","last_unstale":"2023-10-09T04:48:02.042520+0000","last_undegraded":"2023-10-09T04:48:02.042520+0000","last_fullsized":"2023-10-09T04:48:02.042520+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.032005+0000","last_change":"2023-10-09T04:44:51.897117+0000","last_active":"2023-10-09T04:48:02.032005+0000","last_peered":"2023-10-09T04:48:02.032005+0000","last_clean":"2023-10-09T04:48:02.032005+0000","last_became_active":"2023-10-09T04:44:51.896672+0000","last_became_peered":"2023-10-09T04:44:51.896672+0000","last_unstale":"2023-10-09T04:48:02.032005+0000","last_undegraded":"2023-10-09T04:48:02.032005+0000","last_fullsized":"2023-10-09T04:48:02.032005+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:09.426260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":229,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033475+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:48:02.033475+0000","last_peered":"2023-10-09T04:48:02.033475+0000","last_clean":"2023-10-09T04:48:02.033475+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:48:02.033475+0000","last_undegraded":"2023-10-09T04:48:02.033475+0000","last_fullsized":"2023-10-09T04:48:02.033475+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.046848+0000","last_change":"2023-10-09T04:48:02.046557+0000","last_active":"2023-10-09T04:48:02.046848+0000","last_peered":"2023-10-09T04:48:02.046848+0000","last_clean":"2023-10-09T04:48:02.046848+0000","last_became_active":"2023-10-09T04:48:02.046207+0000","last_became_peered":"2023-10-09T04:48:02.046207+0000","last_unstale":"2023-10-09T04:48:02.046848+0000","last_undegraded":"2023-10-09T04:48:02.046848+0000","last_fullsized":"2023-10-09T04:48:02.046848+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.038567+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:48:02.038567+0000","last_peered":"2023-10-09T04:48:02.038567+0000","last_clean":"2023-10-09T04:48:02.038567+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:48:02.038567+0000","last_undegraded":"2023-10-09T04:48:02.038567+0000","last_fullsized":"2023-10-09T04:48:02.038567+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":497,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.031430+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:48:02.031430+0000","last_peered":"2023-10-09T04:48:02.031430+0000","last_clean":"2023-10-09T04:48:02.031430+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:48:02.031430+0000","last_undegraded":"2023-10-09T04:48:02.031430+0000","last_fullsized":"2023-10-09T04:48:02.031430+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.037862+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:48:02.037862+0000","last_peered":"2023-10-09T04:48:02.037862+0000","last_clean":"2023-10-09T04:48:02.037862+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:48:02.037862+0000","last_undegraded":"2023-10-09T04:48:02.037862+0000","last_fullsized":"2023-10-09T04:48:02.037862+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":10,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.157715+0000","last_change":"2023-10-09T04:48:02.156388+0000","last_active":"2023-10-09T04:48:02.157715+0000","last_peered":"2023-10-09T04:48:02.157715+0000","last_clean":"2023-10-09T04:48:02.157715+0000","last_became_active":"2023-10-09T04:48:02.155602+0000","last_became_peered":"2023-10-09T04:48:02.155602+0000","last_unstale":"2023-10-09T04:48:02.157715+0000","last_undegraded":"2023-10-09T04:48:02.157715+0000","last_fullsized":"2023-10-09T04:48:02.157715+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":332,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.036838+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:48:02.036838+0000","last_peered":"2023-10-09T04:48:02.036838+0000","last_clean":"2023-10-09T04:48:02.036838+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:48:02.036838+0000","last_undegraded":"2023-10-09T04:48:02.036838+0000","last_fullsized":"2023-10-09T04:48:02.036838+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033852+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:48:02.033852+0000","last_peered":"2023-10-09T04:48:02.033852+0000","last_clean":"2023-10-09T04:48:02.033852+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:48:02.033852+0000","last_undegraded":"2023-10-09T04:48:02.033852+0000","last_fullsized":"2023-10-09T04:48:02.033852+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040414+0000","last_change":"2023-10-09T04:43:53.980738+0000","last_active":"2023-10-09T04:48:02.040414+0000","last_peered":"2023-10-09T04:48:02.040414+0000","last_clean":"2023-10-09T04:48:02.040414+0000","last_became_active":"2023-10-09T04:43:53.980299+0000","last_became_peered":"2023-10-09T04:43:53.980299+0000","last_unstale":"2023-10-09T04:48:02.040414+0000","last_undegraded":"2023-10-09T04:48:02.040414+0000","last_fullsized":"2023-10-09T04:48:02.040414+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":216,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033348+0000","last_change":"2023-10-09T04:40:47.823824+0000","last_active":"2023-10-09T04:48:02.033348+0000","last_peered":"2023-10-09T04:48:02.033348+0000","last_clean":"2023-10-09T04:48:02.033348+0000","last_became_active":"2023-10-09T04:40:47.817680+0000","last_became_peered":"2023-10-09T04:40:47.817680+0000","last_unstale":"2023-10-09T04:48:02.033348+0000","last_undegraded":"2023-10-09T04:48:02.033348+0000","last_fullsized":"2023-10-09T04:48:02.033348+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:07.850102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":196,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.028559+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:48:02.028559+0000","last_peered":"2023-10-09T04:48:02.028559+0000","last_clean":"2023-10-09T04:48:02.028559+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:48:02.028559+0000","last_undegraded":"2023-10-09T04:48:02.028559+0000","last_fullsized":"2023-10-09T04:48:02.028559+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":177,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042022+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:48:02.042022+0000","last_peered":"2023-10-09T04:48:02.042022+0000","last_clean":"2023-10-09T04:48:02.042022+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:48:02.042022+0000","last_undegraded":"2023-10-09T04:48:02.042022+0000","last_fullsized":"2023-10-09T04:48:02.042022+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":34,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.039843+0000","last_change":"2023-10-09T04:43:53.974915+0000","last_active":"2023-10-09T04:48:02.039843+0000","last_peered":"2023-10-09T04:48:02.039843+0000","last_clean":"2023-10-09T04:48:02.039843+0000","last_became_active":"2023-10-09T04:43:53.974611+0000","last_became_peered":"2023-10-09T04:43:53.974611+0000","last_unstale":"2023-10-09T04:48:02.039843+0000","last_undegraded":"2023-10-09T04:48:02.039843+0000","last_fullsized":"2023-10-09T04:48:02.039843+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":170,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.042398+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:48:02.042398+0000","last_peered":"2023-10-09T04:48:02.042398+0000","last_clean":"2023-10-09T04:48:02.042398+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:48:02.042398+0000","last_undegraded":"2023-10-09T04:48:02.042398+0000","last_fullsized":"2023-10-09T04:48:02.042398+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.040404+0000","last_change":"2023-10-09T04:43:53.972398+0000","last_active":"2023-10-09T04:48:02.040404+0000","last_peered":"2023-10-09T04:48:02.040404+0000","last_clean":"2023-10-09T04:48:02.040404+0000","last_became_active":"2023-10-09T04:43:53.970226+0000","last_became_peered":"2023-10-09T04:43:53.970226+0000","last_unstale":"2023-10-09T04:48:02.040404+0000","last_undegraded":"2023-10-09T04:48:02.040404+0000","last_fullsized":"2023-10-09T04:48:02.040404+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":153,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.036178+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:48:02.036178+0000","last_peered":"2023-10-09T04:48:02.036178+0000","last_clean":"2023-10-09T04:48:02.036178+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:48:02.036178+0000","last_undegraded":"2023-10-09T04:48:02.036178+0000","last_fullsized":"2023-10-09T04:48:02.036178+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.028324+0000","last_change":"2023-10-09T04:44:51.889935+0000","last_active":"2023-10-09T04:48:02.028324+0000","last_peered":"2023-10-09T04:48:02.028324+0000","last_clean":"2023-10-09T04:48:02.028324+0000","last_became_active":"2023-10-09T04:44:51.889421+0000","last_became_peered":"2023-10-09T04:44:51.889421+0000","last_unstale":"2023-10-09T04:48:02.028324+0000","last_undegraded":"2023-10-09T04:48:02.028324+0000","last_fullsized":"2023-10-09T04:48:02.028324+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:53.165469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":245,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.033650+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:48:02.033650+0000","last_peered":"2023-10-09T04:48:02.033650+0000","last_clean":"2023-10-09T04:48:02.033650+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:48:02.033650+0000","last_undegraded":"2023-10-09T04:48:02.033650+0000","last_fullsized":"2023-10-09T04:48:02.033650+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T04:48:02.039748+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:48:02.039748+0000","last_peered":"2023-10-09T04:48:02.039748+0000","last_clean":"2023-10-09T04:48:02.039748+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:48:02.039748+0000","last_undegraded":"2023-10-09T04:48:02.039748+0000","last_fullsized":"2023-10-09T04:48:02.039748+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430687,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596648,"kb_used_data":6944,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93148504,"statfs":{"total":95995035648,"available":95384068096,"internally_reserved":0,"allocated":7110656,"data_stored":6528362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":2.4060000000000001,"15min":2.3380000000000001},"min":{"1min":0.44700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.0309999999999999,"5min":49.149999999999999,"15min":104.16200000000001},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.85399999999999998,"5min":2.3500000000000001,"15min":2.4399999999999999},"min":{"1min":0.53800000000000003,"5min":0.317,"15min":0.27300000000000002},"max":{"1min":1.331,"5min":48.920999999999999,"15min":104.129},"last":0.93400000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":2.5110000000000001,"15min":1.6619999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.0410000000000004,"5min":58.816000000000003,"15min":58.816000000000003},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":2.5179999999999998,"15min":1.776},"min":{"1min":0.38100000000000001,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":3.4740000000000002,"5min":58.840000000000003,"15min":58.840000000000003},"last":1.105}]},{"osd":2,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.601,"15min":1.748},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0620000000000001,"5min":6.5499999999999998,"15min":6.5499999999999998},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.1499999999999999,"15min":2.536},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":3.5270000000000001,"5min":11.583,"15min":11.583},"last":0.84499999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.9179999999999999,"15min":1.7470000000000001},"min":{"1min":0.39800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.5270000000000001,"5min":8.5820000000000007,"15min":9.6869999999999994},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.7430000000000001,"15min":1.5820000000000001},"min":{"1min":0.34200000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.5670000000000002,"5min":12.071999999999999,"15min":12.071999999999999},"last":1.0509999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.6160000000000001,"15min":1.2749999999999999},"min":{"1min":0.33600000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":6.8159999999999998,"5min":11.933999999999999,"15min":11.933999999999999},"last":1.016},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.6659999999999999,"15min":1.3149999999999999},"min":{"1min":0.442,"5min":0.308,"15min":0.308},"max":{"1min":5.8609999999999998,"5min":8.452,"15min":10.696999999999999},"last":0.874}]},{"osd":5,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.706,"15min":1.514},"min":{"1min":0.43099999999999999,"5min":0.41899999999999998,"15min":0.35699999999999998},"max":{"1min":1.4379999999999999,"5min":11.268000000000001,"15min":11.268000000000001},"last":1.177},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.9079999999999999,"15min":1.659},"min":{"1min":0.54100000000000004,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.7679999999999998,"5min":15.138999999999999,"15min":15.138999999999999},"last":0.97399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:47:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.5289999999999999,"15min":1.5069999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.36899999999999999},"max":{"1min":3.6560000000000001,"5min":11.986000000000001,"15min":11.986000000000001},"last":1.222},{"interface":"front","average":{"1min":1.048,"5min":1.7250000000000001,"15min":1.3460000000000001},"min":{"1min":0.47299999999999998,"5min":0.36599999999999999,"15min":0.35599999999999998},"max":{"1min":2.7709999999999999,"5min":12.013999999999999,"15min":12.013999999999999},"last":1.202}]}]},{"osd":6,"up_from":146,"seq":627065225387,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":336992,"kb_used_data":2840,"kb_used_omap":5,"kb_used_meta":334138,"kb_avail":93408160,"statfs":{"total":95995035648,"available":95649955840,"internally_reserved":0,"allocated":2908160,"data_stored":2333931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":342157920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":5.016,"15min":3.6040000000000001},"min":{"1min":0.34599999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.0259999999999998,"5min":214.82300000000001,"15min":214.82300000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.139,"5min":5.2640000000000002,"15min":3.625},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":4.0979999999999999,"5min":213.34700000000001,"15min":213.34700000000001},"last":0.72299999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":3.101,"15min":2.726},"min":{"1min":0.496,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.2380000000000004,"5min":76.081000000000003,"15min":76.081000000000003},"last":0.81499999999999995},{"interface":"front","average":{"1min":1.1990000000000001,"5min":2.573,"15min":2.7250000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.0430000000000001,"5min":75.546000000000006,"15min":75.546000000000006},"last":0.76200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:47:58 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.9119999999999999,"15min":2.0339999999999998},"min":{"1min":0.41699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.0570000000000004,"5min":6.0570000000000004,"15min":6.0570000000000004},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.5189999999999999,"15min":1.554},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":3.7360000000000002,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.63400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.712,"15min":1.7969999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.9590000000000001,"5min":6.7599999999999998,"15min":11.673},"last":1.034},{"interface":"front","average":{"1min":1.575,"5min":1.6000000000000001,"15min":1.8380000000000001},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.6479999999999997,"5min":9.3190000000000008,"15min":15.824},"last":0.60099999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.712,"15min":1.3620000000000001},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.3450000000000002,"5min":13.699999999999999,"15min":13.699999999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.601,"5min":1.6799999999999999,"15min":1.46},"min":{"1min":0.47999999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":10.324},"last":0.99399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.726,"15min":1.5940000000000001},"min":{"1min":0.57999999999999996,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.9940000000000002,"5min":9.3000000000000007,"15min":10.193},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.778,"15min":1.706},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7989999999999999,"5min":9.9740000000000002,"15min":11.727},"last":1.0860000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.8620000000000001,"15min":1.677},"min":{"1min":0.432,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.6109999999999998,"5min":16.527999999999999,"15min":16.527999999999999},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.883,"15min":1.5780000000000001},"min":{"1min":0.38900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.2510000000000003,"5min":15.366,"15min":15.366},"last":1.0009999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958836,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547312,"kb_used_data":2728,"kb_used_omap":6,"kb_used_meta":544569,"kb_avail":93197840,"statfs":{"total":95995035648,"available":95434588160,"internally_reserved":0,"allocated":2793472,"data_stored":1796934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":557638668},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":2.5750000000000002,"15min":2.806},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.17499999999999999},"max":{"1min":4.2430000000000003,"5min":41.305,"15min":91.025999999999996},"last":0.751},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.6349999999999998,"15min":2.9100000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.17999999999999999},"max":{"1min":5.734,"5min":43.777999999999999,"15min":91.536000000000001},"last":0.61899999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:47:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":2.4990000000000001,"15min":2.637},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.153,"5min":15.336,"15min":15.336},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.1440000000000001,"15min":2.4119999999999999},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":5.6580000000000004,"5min":13.444000000000001,"15min":13.444000000000001},"last":0.64500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.6299999999999999,"15min":1.8140000000000001},"min":{"1min":0.32900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.0940000000000003,"5min":10.401999999999999,"15min":14.369},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.544,"5min":1.778,"15min":1.8620000000000001},"min":{"1min":0.29699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.9410000000000007,"5min":19.617999999999999,"15min":19.617999999999999},"last":0.79100000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":1.788,"15min":1.462},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.7639999999999993,"5min":13.362,"15min":13.362},"last":0.81299999999999994},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.726,"15min":1.488},"min":{"1min":0.39200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.1319999999999997,"5min":8.6890000000000001,"15min":12.209},"last":0.91900000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.9950000000000001,"15min":1.7290000000000001},"min":{"1min":0.55600000000000005,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":5.6920000000000002,"5min":14.494999999999999,"15min":14.494999999999999},"last":0.877},{"interface":"front","average":{"1min":0.84099999999999997,"5min":2.008,"15min":1.663},"min":{"1min":0.371,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":1.5509999999999999,"5min":13.413,"15min":13.413},"last":0.72899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.909,"15min":1.859},"min":{"1min":0.432,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.7649999999999997,"5min":14.545,"15min":14.545},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.788,"5min":2.0550000000000002,"15min":1.9370000000000001},"min":{"1min":0.52200000000000002,"5min":0.40500000000000003,"15min":0.39900000000000002},"max":{"1min":5.7969999999999997,"5min":14.577,"15min":14.577},"last":1.133}]},{"osd":7,"last update":"Mon Oct 9 04:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":2.02,"15min":2.0499999999999998},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.6719999999999997,"5min":13.474,"15min":144.613},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.748,"15min":1.994},"min":{"1min":0.57299999999999995,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.1639999999999997,"5min":8.6500000000000004,"15min":145.53700000000001},"last":1.2150000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122361,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":612868,"kb_used_data":3324,"kb_used_omap":9,"kb_used_meta":609526,"kb_avail":93132284,"statfs":{"total":95995035648,"available":95367458816,"internally_reserved":0,"allocated":3403776,"data_stored":2392699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":624155121},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:47:47 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.4690000000000001,"15min":2.2400000000000002},"min":{"1min":0.29799999999999999,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":4.2729999999999997,"5min":7.5270000000000001,"15min":223.98500000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.8500000000000001,"15min":2.3679999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19900000000000001},"max":{"1min":5.3840000000000003,"5min":10.500999999999999,"15min":223.13},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.4020000000000001,"15min":2.4470000000000001},"min":{"1min":0.36499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.7850000000000001,"5min":17.109000000000002,"15min":17.109000000000002},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.294,"5min":2.629,"15min":2.7949999999999999},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.202,"5min":12.57,"15min":12.57},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:47:47 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.5640000000000001,"15min":1.8560000000000001},"min":{"1min":0.33800000000000002,"5min":0.26700000000000002,"15min":0.23400000000000001},"max":{"1min":5.8330000000000002,"5min":10.933999999999999,"15min":64.622},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.974,"5min":1.976,"15min":2.0840000000000001},"min":{"1min":0.309,"5min":0.23999999999999999,"15min":0.218},"max":{"1min":7.2779999999999996,"5min":10.398999999999999,"15min":64.540999999999997},"last":0.45900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.9099999999999999,"15min":1.734},"min":{"1min":0.52400000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.649,"5min":16.84,"15min":61.549999999999997},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.7450000000000001,"15min":1.569},"min":{"1min":0.67100000000000004,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":1.833,"5min":39.957999999999998,"15min":61.902999999999999},"last":0.872}]},{"osd":5,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.762,"15min":2.1280000000000001},"min":{"1min":0.5,"5min":0.35399999999999998,"15min":0.311},"max":{"1min":7.1500000000000004,"5min":7.1500000000000004,"15min":61.972999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.5109999999999999,"15min":2.1110000000000002},"min":{"1min":0.621,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.5720000000000001,"5min":10.215999999999999,"15min":62.417000000000002},"last":0.84499999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:47:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.4630000000000001,"15min":2.153},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6559999999999997,"5min":7.5759999999999996,"15min":62.462000000000003},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.8700000000000001,"15min":2.4260000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.32100000000000001},"max":{"1min":7.2279999999999998,"5min":9.1969999999999992,"15min":61.808},"last":1.004}]},{"osd":7,"last update":"Mon Oct 9 04:48:14 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.9710000000000001,"15min":1.9179999999999999},"min":{"1min":0.73499999999999999,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":1.7509999999999999,"5min":7.2519999999999998,"15min":62.006},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.931,"15min":1.8819999999999999},"min":{"1min":0.63100000000000001,"5min":0.372,"15min":0.32300000000000001},"max":{"1min":2.0219999999999998,"5min":10.804,"15min":61.606999999999999},"last":0.98699999999999999}]}]},{"osd":2,"up_from":234,"seq":1005022347307,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":203704,"kb_used_data":6832,"kb_used_omap":13,"kb_used_meta":196850,"kb_avail":93541448,"statfs":{"total":95995035648,"available":95786442752,"internally_reserved":0,"allocated":6995968,"data_stored":5992677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":201575295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.5609999999999999,"15min":2.9569999999999999},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.754,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":2.4060000000000001,"15min":2.8650000000000002},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.249},"max":{"1min":0.84199999999999997,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.57399999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.427,"15min":2.5670000000000002},"min":{"1min":0.31900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7619999999999996,"5min":7.907,"15min":7.907},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.198,"5min":1.7949999999999999,"15min":1.877},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8780000000000001,"5min":7.8780000000000001,"15min":7.8780000000000001},"last":0.84699999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.6859999999999999,"15min":1.673},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.0720000000000001,"5min":14.340999999999999,"15min":14.340999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.2649999999999999,"5min":2.4870000000000001,"15min":2.7450000000000001},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.4550000000000001,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":2.0339999999999998,"15min":2.137},"min":{"1min":0.47799999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6779999999999999,"5min":6.6520000000000001,"15min":6.6520000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.994,"15min":2.0459999999999998},"min":{"1min":0.40999999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":3.3769999999999998,"5min":7.9390000000000001,"15min":7.9390000000000001},"last":1.1850000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.9630000000000001,"15min":2.1659999999999999},"min":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.4910000000000001,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.546,"5min":2.6509999999999998,"15min":2.9039999999999999},"min":{"1min":0.68400000000000005,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":3.706,"5min":12.295999999999999,"15min":12.295999999999999},"last":0.97699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.1389999999999998,"15min":2.2749999999999999},"min":{"1min":0.54700000000000004,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":3.125,"5min":8.2409999999999997,"15min":8.2409999999999997},"last":1.2709999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.724,"15min":1.7070000000000001},"min":{"1min":0.436,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.6240000000000001,"5min":7.3040000000000003,"15min":7.3040000000000003},"last":1.3029999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":2.3359999999999999,"15min":2.3039999999999998},"min":{"1min":0.56699999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":14.161,"5min":14.161,"15min":14.161},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.365,"5min":2.0859999999999999,"15min":2.0539999999999998},"min":{"1min":0.56799999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":7.2709999999999999,"5min":10.996,"15min":10.996},"last":1.1299999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631782,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":638204,"kb_used_data":2740,"kb_used_omap":5,"kb_used_meta":635450,"kb_avail":93106948,"statfs":{"total":95995035648,"available":95341514752,"internally_reserved":0,"allocated":2805760,"data_stored":1799429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":650700974},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.4710000000000001,"15min":2.8340000000000001},"min":{"1min":0.375,"5min":0.19500000000000001,"15min":0.154},"max":{"1min":6.8730000000000002,"5min":8.5220000000000002,"15min":341.28699999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.4650000000000001,"15min":2.7890000000000001},"min":{"1min":0.41299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":7.3440000000000003,"5min":9.5079999999999991,"15min":341.57799999999997},"last":0.60199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.444,"15min":1.679},"min":{"1min":0.22,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":17.172999999999998,"5min":17.172999999999998,"15min":47.637999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.4289999999999998,"5min":1.9259999999999999,"15min":1.901},"min":{"1min":0.22500000000000001,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":17.125,"5min":17.125,"15min":47.616999999999997},"last":0.69899999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:47:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.591,"15min":1.663},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7409999999999997,"5min":6.4240000000000004,"15min":6.4240000000000004},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.8220000000000001,"15min":1.8959999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3150000000000004,"5min":10.154999999999999,"15min":10.154999999999999},"last":0.73799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.4379999999999999,"15min":1.411},"min":{"1min":0.39800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":2.4660000000000002,"5min":8.4350000000000005,"15min":12.981},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.5009999999999999,"5min":1.7549999999999999,"15min":1.4890000000000001},"min":{"1min":0.44,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":7.2469999999999999,"5min":9.4359999999999999,"15min":9.4359999999999999},"last":0.98899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":2.0329999999999999,"15min":1.728},"min":{"1min":0.53400000000000003,"5min":0.35399999999999998,"15min":0.32300000000000001},"max":{"1min":7.2859999999999996,"5min":7.9649999999999999,"15min":8.0109999999999992},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.427,"5min":1.746,"15min":1.742},"min":{"1min":0.626,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":7.21,"5min":9.4800000000000004,"15min":9.4800000000000004},"last":1.075}]},{"osd":6,"last update":"Mon Oct 9 04:47:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":1.3440000000000001,"15min":1.7729999999999999},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":9.0020000000000007,"5min":9.0020000000000007,"15min":11.842000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.341,"5min":1.504,"15min":1.623},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.4489999999999998,"5min":8.4860000000000007,"15min":13.004},"last":1.0009999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.6539999999999999,"15min":2.5329999999999999},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.9969999999999999,"5min":6.7729999999999997,"15min":198.012},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.633,"15min":2.472},"min":{"1min":0.42799999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.669,"5min":8.4469999999999992,"15min":198.05199999999999},"last":1.0309999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019384,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":604256,"kb_used_data":2264,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":93140896,"statfs":{"total":95995035648,"available":95376277504,"internally_reserved":0,"allocated":2318336,"data_stored":1743620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":2.52,"15min":1.738},"min":{"1min":0.49299999999999999,"5min":0.26500000000000001,"15min":0.26100000000000001},"max":{"1min":6.6600000000000001,"5min":52.106999999999999,"15min":52.106999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.86599999999999999,"5min":2.5449999999999999,"15min":1.9930000000000001},"min":{"1min":0.49199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":1.262,"5min":52.289999999999999,"15min":52.289999999999999},"last":0.74399999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:47:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":1.837,"15min":1.758},"min":{"1min":0.379,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.0030000000000001,"5min":24.329999999999998,"15min":79.361999999999995},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.375,"5min":2.3319999999999999,"15min":1.8700000000000001},"min":{"1min":0.33500000000000002,"5min":0.315,"15min":0.30399999999999999},"max":{"1min":8.0229999999999997,"5min":24.741,"15min":79.292000000000002},"last":0.92600000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:47:59 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.9059999999999999,"15min":2.0059999999999998},"min":{"1min":0.48999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.249000000000001,"5min":10.249000000000001,"15min":10.249000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.556,"15min":2.4630000000000001},"min":{"1min":0.55200000000000005,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.535,"5min":10.535,"15min":10.535},"last":0.84599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.637,"15min":1.5169999999999999},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.6219999999999999,"5min":10.42,"15min":10.42},"last":0.999},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.6319999999999999,"15min":1.464},"min":{"1min":0.42599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0220000000000002,"5min":8.4909999999999997,"15min":9.484},"last":1.1279999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":2.1829999999999998,"15min":1.385},"min":{"1min":0.58599999999999997,"5min":0.251,"15min":0.251},"max":{"1min":6.7249999999999996,"5min":9.0549999999999997,"15min":10.412000000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.7969999999999999,"15min":1.401},"min":{"1min":0.39200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.4179999999999999,"5min":8.9209999999999994,"15min":8.9209999999999994},"last":0.94499999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:47:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.72,"15min":1.252},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.9029999999999996,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.546,"15min":1.266},"min":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.6150000000000002,"5min":8.5470000000000006,"15min":10.257999999999999},"last":1.0580000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:47:33 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.615,"15min":1.2290000000000001},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.317},"max":{"1min":6.8129999999999997,"5min":7.4029999999999996,"15min":9.0370000000000008},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.573,"5min":1.6679999999999999,"15min":1.379},"min":{"1min":0.51500000000000001,"5min":0.35099999999999998,"15min":0.30399999999999999},"max":{"1min":6.8689999999999998,"5min":8.9670000000000005,"15min":13.349},"last":1.1699999999999999}]}]},{"osd":5,"up_from":150,"seq":644245094566,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351952,"kb_used_data":6344,"kb_used_omap":6,"kb_used_meta":345593,"kb_avail":93393200,"statfs":{"total":95995035648,"available":95634636800,"internally_reserved":0,"allocated":6496256,"data_stored":5933372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":353887507},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.9870000000000001,"15min":1.698},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8809999999999998,"5min":19.001000000000001,"15min":19.001000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.94099999999999995,"5min":2.1760000000000002,"15min":1.8859999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30099999999999999},"max":{"1min":2.6379999999999999,"5min":18.119,"15min":18.119},"last":0.748}]},{"osd":1,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.7110000000000001,"15min":1.6739999999999999},"min":{"1min":0.55000000000000004,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.3370000000000002,"5min":9.4309999999999992,"15min":9.4309999999999992},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.04,"5min":1.879,"15min":1.7889999999999999},"min":{"1min":0.496,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9529999999999998,"5min":11.311999999999999,"15min":11.311999999999999},"last":0.96499999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:47:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.714,"15min":1.7050000000000001},"min":{"1min":0.47699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.7670000000000003,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.585,"5min":2.0870000000000002,"15min":2.327},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9139999999999997,"5min":13.536,"15min":13.536},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.5820000000000001,"15min":1.573},"min":{"1min":0.56499999999999995,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.1190000000000002,"5min":9.3889999999999993,"15min":9.3889999999999993},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.8999999999999999,"15min":1.9419999999999999},"min":{"1min":0.47399999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.7559999999999998,"5min":11.273,"15min":11.273},"last":0.878}]},{"osd":4,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.7929999999999999,"15min":1.542},"min":{"1min":0.44,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.7569999999999999,"5min":11.359,"15min":11.359},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.89200000000000002,"5min":1.704,"15min":1.3600000000000001},"min":{"1min":0.32300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.349,"5min":10.709,"15min":10.709},"last":0.84299999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.784,"15min":1.52},"min":{"1min":0.38600000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.375,"5min":10.77,"15min":11.087},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.081,"5min":1.8049999999999999,"15min":1.5780000000000001},"min":{"1min":0.66900000000000004,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.298,"5min":11.222,"15min":11.222},"last":0.66700000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.8280000000000001,"15min":1.804},"min":{"1min":0.73499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.9690000000000001,"5min":11.185,"15min":39.771000000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.02,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3140000000000001,"5min":14.170999999999999,"15min":39.905000000000001},"last":0.89500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:48:24.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:48:24.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:24.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:24.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:24.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:24.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:48:25.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:25.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:25.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:25.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: audit 2023-10-09T04:48:23.626390+0000 mgr.y (mgr.34107) 1029 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.083328+0000 mon.a (mon.0) 1521 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.096510+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: audit 2023-10-09T04:48:24.100389+0000 mon.a (mon.0) 1522 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: audit 2023-10-09T04:48:24.100525+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.100633+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:48:25.188 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.863951+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: audit 2023-10-09T04:48:23.626390+0000 mgr.y (mgr.34107) 1029 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.083328+0000 mon.a (mon.0) 1521 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.096510+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: audit 2023-10-09T04:48:24.100389+0000 mon.a (mon.0) 1522 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: audit 2023-10-09T04:48:24.100525+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.100633+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:48:25.189 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:25 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.863951+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:25.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:25.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: audit 2023-10-09T04:48:23.626390+0000 mgr.y (mgr.34107) 1029 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.083328+0000 mon.a (mon.0) 1521 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.096510+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: audit 2023-10-09T04:48:24.100389+0000 mon.a (mon.0) 1522 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:48:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: audit 2023-10-09T04:48:24.100525+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.100633+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:48:25.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:25 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.863951+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:25.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:25.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:25.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:25.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:25.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:25.697 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:25.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:25.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:25.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:25.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:25.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:25.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:25.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:25.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:26.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:26.121 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:26.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:26.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.701667+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.864285+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[77063]: cluster 2023-10-09T04:48:24.864409+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.701667+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.864285+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:26 smithi082 bash[79733]: cluster 2023-10-09T04:48:24.864409+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:26.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:26.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:26.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:26 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.701667+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:26.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:26 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.864285+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:26 smithi153 bash[59565]: cluster 2023-10-09T04:48:24.864409+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T04:48:26.393 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:26.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:26.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:26.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:26.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:26.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:26.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:26.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:26.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:26.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:27.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:27.150 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T04:48:27.153 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:48:27.153 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:48:27.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:27.199 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:48:27.199 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:48:27.199 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_200616 2023-10-09T04:48:27.200 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:48:27.200 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:48:27.201 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:27.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:27.333 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:48:27.333 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:48:27.333 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:48:27.333 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:48:27.333 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:89915 8.0 8:0bea098e:::benchmark_data_smithi082_175677_object89914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:43:39.260118+0000", 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 288.06195214600001, 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15377838699999999, 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 89915 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.334 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260118+0000", 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260118+0000", 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.335 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260116+0000", 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1076999999999999e-05 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260167+0000", 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0828e-05 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260178+0000", 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1789999999999999e-06 2023-10-09T04:48:27.336 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260183+0000", 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0094298230000000004 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.269613+0000", 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000121334 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.337 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.269734+0000", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014708500000000001 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.269882+0000", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0032870159999999998 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.273169+0000", 2023-10-09T04:48:27.338 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002535341 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.275704+0000", 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.138052119 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.413756+0000", 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0680999999999995e-05 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.339 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.413837+0000", 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9781000000000002e-05 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.413896+0000", 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15377838699999999 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.340 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:89916 8.0 8:0d452845:::benchmark_data_smithi082_175677_object89915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e225)", 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:43:39.260190+0000", 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 288.06188014600002, 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15383559799999999, 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 89916 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.341 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260190+0000", 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260190+0000", 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.342 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260189+0000", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9990000000000001e-05 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260239+0000", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7389999999999999e-06 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260247+0000", 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1564e-05 2023-10-09T04:48:27.343 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.260259+0000", 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.012998294000000001 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.273257+0000", 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014866900000000001 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.344 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.273406+0000", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000109836 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.273515+0000", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023995399999999999 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.275915+0000", 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018988410000000001 2023-10-09T04:48:27.345 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.277814+0000", 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13614116900000001 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.413955+0000", 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0958000000000003e-05 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.346 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.413996+0000", 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.968e-05 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:43:39.414026+0000", 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15383559799999999 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.347 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112218 8.0 8:0140851c:::benchmark_data_smithi082_175677_object112217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:03.597728+0000", 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 263.72434249600002, 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21243753700000001, 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112218 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.348 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597728+0000", 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597728+0000", 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.349 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597726+0000", 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0453000000000001e-05 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597757+0000", 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0438000000000001e-05 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597767+0000", 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1059999999999996e-06 2023-10-09T04:48:27.350 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597773+0000", 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4216999999999999e-05 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597827+0000", 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000156811 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.351 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.597984+0000", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015565800000000001 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.598140+0000", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0038813530000000001 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.602021+0000", 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0056662220000000003 2023-10-09T04:48:27.352 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.607687+0000", 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.20229734999999999 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.809985+0000", 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010597100000000001 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.353 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.810091+0000", 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4475999999999996e-05 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:03.810165+0000", 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.21243753700000001 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.354 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:112912 8.0 8:0d48ad63:::benchmark_data_smithi082_175677_object112911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:44:04.582443+0000", 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 262.73962730400001, 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.152355711, 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 112912 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.355 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582443+0000", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582443+0000", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582441+0000", 2023-10-09T04:48:27.356 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2392999999999998e-05 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582494+0000", 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7034e-05 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582511+0000", 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9629999999999993e-06 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.357 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582521+0000", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6882000000000001e-05 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582587+0000", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000133523 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582721+0000", 2023-10-09T04:48:27.358 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014772099999999999 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.582869+0000", 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0085803879999999996 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.591449+0000", 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049332450000000002 2023-10-09T04:48:27.359 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.596382+0000", 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13821424299999999 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.734597+0000", 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000117012 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.360 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.734714+0000", 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5086000000000005e-05 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:44:04.734799+0000", 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.152355711 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:157899 8.0 8:0a5e74df:::benchmark_data_smithi082_175677_object40885:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:48:27.361 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:08.367483+0000", 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.95458737600001, 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19213769999999999, 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 157899 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.362 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367483+0000", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367483+0000", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367481+0000", 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5329999999999994e-06 2023-10-09T04:48:27.363 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367490+0000", 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5985e-05 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367506+0000", 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9160000000000007e-06 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.364 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367514+0000", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3922000000000001e-05 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367598+0000", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018718599999999999 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367786+0000", 2023-10-09T04:48:27.365 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000238027 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.368024+0000", 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.016043691999999998 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384067+0000", 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0083004129999999995 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.366 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.392368+0000", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16710640600000001 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.559474+0000", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4374000000000003e-05 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.559548+0000", 2023-10-09T04:48:27.367 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2082999999999994e-05 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.559621+0000", 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.19213769999999999 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:157900 8.8 8:1860b255:::benchmark_data_smithi082_175677_object40886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:48:27.368 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:08.367593+0000", 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.95447696700001, 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.194744162, 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 157900 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.369 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367593+0000", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367593+0000", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367593+0000", 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.087e-06 2023-10-09T04:48:27.370 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367597+0000", 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1137e-05 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367608+0000", 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3509999999999996e-06 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.371 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.367615+0000", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00050415499999999995 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.368119+0000", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012860399999999999 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.368248+0000", 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013846600000000001 2023-10-09T04:48:27.372 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.368386+0000", 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.015650285 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384037+0000", 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9449999999999993e-06 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.373 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384045+0000", 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.178179528 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562224+0000", 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0201000000000003e-05 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562274+0000", 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3113000000000004e-05 2023-10-09T04:48:27.374 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562337+0000", 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.194744162 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:157908 8.8 8:1f3049fd:::benchmark_data_smithi082_175677_object40894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:08.378881+0000", 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.94318947900001, 2023-10-09T04:48:27.375 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183599172, 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 157908 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378881+0000", 2023-10-09T04:48:27.376 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378881+0000", 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378879+0000", 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2840000000000001e-06 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.377 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378887+0000", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4063000000000001e-05 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378901+0000", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9410000000000001e-06 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.378908+0000", 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0051764489999999996 2023-10-09T04:48:27.378 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384084+0000", 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013091599999999999 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384215+0000", 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000105932 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.379 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384321+0000", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080364689999999992 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.392358+0000", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00050973399999999997 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.392867+0000", 2023-10-09T04:48:27.380 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16950959600000001 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562377+0000", 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8769e-05 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562426+0000", 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4292999999999997e-05 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.381 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562480+0000", 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.183599172 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:157911 8.0 8:0a3eb17a:::benchmark_data_smithi082_175677_object40897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:08.383508+0000", 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 198.938561983, 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17908153800000001, 2023-10-09T04:48:27.382 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 157911 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383508+0000", 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.383 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383508+0000", 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383507+0000", 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1240000000000002e-06 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.384 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383513+0000", 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0993e-05 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383524+0000", 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7209999999999999e-06 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.383530+0000", 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00057095099999999999 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.385 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384101+0000", 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013098799999999999 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384232+0000", 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013179700000000001 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.386 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.384364+0000", 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0080500820000000001 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.392414+0000", 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00046230700000000002 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.392876+0000", 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.169513898 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.387 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562390+0000", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010924700000000001 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562499+0000", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0431000000000002e-05 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:08.562590+0000", 2023-10-09T04:48:27.388 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17908153800000001 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:161298 8.4 8:2ae2ed88:::benchmark_data_smithi082_175677_object44284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:45:13.944151+0000", 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 193.37791897400001, 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16065791700000001, 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.389 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 161298 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944151+0000", 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.390 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944151+0000", 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944151+0000", 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9330000000000003e-06 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944154+0000", 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6459999999999993e-06 2023-10-09T04:48:27.391 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944162+0000", 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2819999999999998e-06 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.944166+0000", 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.020331821 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.392 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.964498+0000", 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018347 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.964682+0000", 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000210471 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.964892+0000", 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.010436399000000001 2023-10-09T04:48:27.393 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.975329+0000", 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0054078950000000002 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:13.980736+0000", 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.11978499400000001 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.394 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:14.100521+0000", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1658999999999998e-05 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:14.100613+0000", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0041960369999999997 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:45:14.104809+0000", 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16065791700000001 2023-10-09T04:48:27.395 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190668 8.0 8:05c08a12:::benchmark_data_smithi082_175677_object73654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.801429+0000", 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 144.52064093600001, 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16165960700000001, 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.396 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190668 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801429+0000", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801429+0000", 2023-10-09T04:48:27.397 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801428+0000", 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801436+0000", 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3298e-05 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.398 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801449+0000", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5499999999999997e-06 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801457+0000", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2106000000000005e-05 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801519+0000", 2023-10-09T04:48:27.399 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000115909 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801635+0000", 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016839900000000001 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.801803+0000", 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011168854000000001 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.400 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.812972+0000", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8989999999999998e-06 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.812978+0000", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15001167600000001 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.962989+0000", 2023-10-09T04:48:27.401 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0609000000000003e-05 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.963040+0000", 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8871e-05 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.963089+0000", 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16165960700000001 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.402 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:190670 8.0 8:0467d5e4:::benchmark_data_smithi082_175677_object73656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:02.804517+0000", 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 144.51755320699999, 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16066181700000001, 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.403 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 190670 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804517+0000", 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804517+0000", 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.404 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804516+0000", 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5447999999999998e-05 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804531+0000", 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1325000000000001e-05 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.405 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804543+0000", 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6429999999999998e-06 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.804549+0000", 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022541660000000002 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806804+0000", 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015800300000000001 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.406 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.806962+0000", 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000197154 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.807159+0000", 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0058567419999999999 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.407 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.813015+0000", 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000110235 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.813126+0000", 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15193869800000001 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.965064+0000", 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7039000000000006e-05 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.408 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.965141+0000", 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7388000000000002e-05 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:02.965179+0000", 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16066181700000001 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.409 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:197803 8.0 8:074d8910:::benchmark_data_smithi082_175677_object80789:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:14.708115+0000", 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 132.61395539099999, 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16726011199999999, 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 197803 2023-10-09T04:48:27.410 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708115+0000", 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708115+0000", 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999971 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.411 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708112+0000", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5393e-05 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708127+0000", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7784000000000003e-05 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708165+0000", 2023-10-09T04:48:27.412 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5480000000000001e-05 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708181+0000", 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016518899999999999 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708346+0000", 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00026809499999999999 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.413 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708614+0000", 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000296483 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.708910+0000", 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.006176618 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.414 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.715087+0000", 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100747 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.715188+0000", 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16003421200000001 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875222+0000", 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.1910999999999995e-05 2023-10-09T04:48:27.415 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875314+0000", 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1063999999999997e-05 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875375+0000", 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16726011199999999 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.416 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:197805 8.0 8:043e560d:::benchmark_data_smithi082_175677_object80791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:14.716962+0000", 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 132.605107933, 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15855593700000001, 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 197805 2023-10-09T04:48:27.417 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716962+0000", 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716962+0000", 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.418 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716960+0000", 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2599999999999994e-06 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716969+0000", 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6594000000000001e-05 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.419 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716986+0000", 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4370000000000001e-06 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.716993+0000", 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4020999999999997e-05 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.717037+0000", 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100707 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.420 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.717138+0000", 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000105022 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.717243+0000", 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0048316610000000001 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.421 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.722075+0000", 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0021101050000000001 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.724185+0000", 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15125720100000001 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875442+0000", 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1819000000000001e-05 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.422 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875484+0000", 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4430000000000001e-05 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:14.875518+0000", 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15855593700000001 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.423 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:224130 8.8 8:18db9d1d:::benchmark_data_smithi082_175677_object107116:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:57.762263+0000", 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 89.559806739999999, 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17417534600000001, 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 224130 2023-10-09T04:48:27.424 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762263+0000", 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762263+0000", 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.425 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762263+0000", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1869999999999996e-06 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762268+0000", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2739999999999996e-06 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762277+0000", 2023-10-09T04:48:27.426 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0789999999999997e-06 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.762283+0000", 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049143399999999997 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767197+0000", 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000115957 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.427 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767313+0000", 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013278699999999999 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767446+0000", 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0077946259999999998 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.428 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.775241+0000", 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.018494769000000001 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.793736+0000", 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.142581288 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936317+0000", 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1325e-05 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.429 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936378+0000", 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0652000000000001e-05 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936439+0000", 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17417534600000001 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.430 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:224131 8.0 8:033bea78:::benchmark_data_smithi082_175677_object107117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:57.764774+0000", 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 89.557296610999998, 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17147436299999999, 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 224131 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.431 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764774+0000", 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764774+0000", 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.432 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764772+0000", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8449999999999992e-06 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764781+0000", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6184e-05 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764797+0000", 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0729999999999999e-06 2023-10-09T04:48:27.433 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.764804+0000", 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023758350000000002 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767180+0000", 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9264000000000005e-05 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.434 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767269+0000", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2802999999999999e-05 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.767362+0000", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0078721030000000001 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.775234+0000", 2023-10-09T04:48:27.435 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0071604540000000001 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.782394+0000", 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.153412415 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.935807+0000", 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000373247 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.436 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936180+0000", 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7890000000000002e-05 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936248+0000", 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.17147436299999999 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.437 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:224137 8.0 8:0657ca2a:::benchmark_data_smithi082_175677_object107123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:57.776231+0000", 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 89.545838845999995, 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16037538800000001, 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 224137 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.438 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776231+0000", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776231+0000", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776229+0000", 2023-10-09T04:48:27.439 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8320000000000001e-06 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776239+0000", 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6271999999999999e-05 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776255+0000", 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0509999999999999e-06 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.440 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776262+0000", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5016999999999999e-05 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776317+0000", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014357700000000001 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776461+0000", 2023-10-09T04:48:27.441 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015004100000000001 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.776611+0000", 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0056928889999999996 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.782304+0000", 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011414802 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.442 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.793719+0000", 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.14277135199999999 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936490+0000", 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9409e-05 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.443 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936549+0000", 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7339000000000001e-05 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936607+0000", 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.16037538800000001 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:224147 8.8 8:1705d07c:::benchmark_data_smithi082_175677_object107133:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T04:48:27.444 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:57.783928+0000", 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 89.538142391999997, 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15286169499999999, 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 224147 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.445 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783928+0000", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783928+0000", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783926+0000", 2023-10-09T04:48:27.446 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9809999999999992e-06 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783935+0000", 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8028000000000001e-05 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783953+0000", 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7619999999999998e-06 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.447 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.783960+0000", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9746000000000001e-05 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.784009+0000", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015033000000000001 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.784160+0000", 2023-10-09T04:48:27.448 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0041339849999999997 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.788294+0000", 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0056894800000000002 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.793983+0000", 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0046128219999999999 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.449 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.798596+0000", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.13807783300000001 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936674+0000", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9602000000000003e-05 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936733+0000", 2023-10-09T04:48:27.450 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6104999999999999e-05 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:57.936790+0000", 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.15286169499999999 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:48:27.451 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:48:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.452 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:27.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:27.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.581 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:27.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:27.714 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:27.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:27.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:27.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:27.944 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:27.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:27.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:28.077 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:28.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:28.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:48:28.200 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 16 2887 2871 179.409 179.438 0.00742227 0.00555022 2023-10-09T04:48:28.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:28.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:28 smithi082 bash[77063]: cluster 2023-10-09T04:48:26.702883+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1101: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:28.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:28 smithi082 bash[79733]: cluster 2023-10-09T04:48:26.702883+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1101: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:28.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:28.341 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:28.355 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:28 smithi153 bash[59565]: cluster 2023-10-09T04:48:26.702883+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1101: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:48:28.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:48:28.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:28.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:28.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:28.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:28.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:28.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:28.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:28.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:28.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:28.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:28.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:28.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:28.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:29.023 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:29.157 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.186 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:29.200 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 16 5480 5464 170.729 162.062 0.00460862 0.00584455 2023-10-09T04:48:29.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:29.326 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:29.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:29.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:29.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:29.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:29.654 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:29.654 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:29.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:29.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:29.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.853 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:29.957 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:29.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:29] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.33.4" 2023-10-09T04:48:29.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:29.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:30.097 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:30.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:30.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 15 8095 8080 168.314 163.5 0.00465738 0.00593226 2023-10-09T04:48:30.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:30.241 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:30 smithi153 bash[59565]: cluster 2023-10-09T04:48:28.703499+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 20 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T04:48:30.241 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:30 smithi153 bash[59565]: audit 2023-10-09T04:48:29.298710+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:30.242 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:30 smithi153 bash[59565]: audit 2023-10-09T04:48:29.299996+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:30.262 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:30.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:30.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[77063]: cluster 2023-10-09T04:48:28.703499+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 20 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T04:48:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[77063]: audit 2023-10-09T04:48:29.298710+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[77063]: audit 2023-10-09T04:48:29.299996+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[79733]: cluster 2023-10-09T04:48:28.703499+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 20 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T04:48:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[79733]: audit 2023-10-09T04:48:29.298710+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:48:30.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:30 smithi082 bash[79733]: audit 2023-10-09T04:48:29.299996+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:30.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:30.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:30.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:30.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:30.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:30.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:30.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:30.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:30.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:30.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:31.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:31.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:31.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:31.201 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 16 10610 10594 165.513 157.125 0.00352403 0.00603335 2023-10-09T04:48:31.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:31.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:31.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:31.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:31.473 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:31.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:31.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:31.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:31.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:31.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:31.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:31.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:31.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:31.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:31.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:32.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:32.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:32.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:32.201 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 12908 12892 161.133 143.625 0.0093618 0.00619432 2023-10-09T04:48:32.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:32.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:32 smithi082 bash[77063]: cluster 2023-10-09T04:48:30.704331+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 368 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:48:32.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:32 smithi082 bash[79733]: cluster 2023-10-09T04:48:30.704331+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 368 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:48:32.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:32.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:32.382 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:32 smithi153 bash[59565]: cluster 2023-10-09T04:48:30.704331+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 368 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:48:32.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:32.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:32.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:32.644 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:32.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:32.709 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:32.710 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:32.748 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:32.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:32.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:32.907 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:32.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:33.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:33.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:33.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:33.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:33.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:33.201 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 16 15084 15068 156.942 136 0.00743913 0.0063652 2023-10-09T04:48:33.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:33.286 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:33 smithi082 bash[77063]: cluster 2023-10-09T04:48:32.704863+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 480 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:48:33.287 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:33 smithi082 bash[79733]: cluster 2023-10-09T04:48:32.704863+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 480 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:48:33.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:33.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:33.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:33 smithi153 bash[59565]: cluster 2023-10-09T04:48:32.704863+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 480 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:48:33.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:33.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:33.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:33.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:33.666 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:33.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:33.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:33.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:33.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:33.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:33.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:34.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:34.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:34.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:34.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 17349 17333 154.719 141.562 0.00621699 0.00645604 2023-10-09T04:48:34.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:34.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:34.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:34.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:34.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:34.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:34.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:34.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:34.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:34.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:34.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:34.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:34.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:34.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:35.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:35.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:35.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:35.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:35.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 19502 19486 152.198 134.562 0.00586364 0.00656427 2023-10-09T04:48:35.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:35.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:35.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:35.385 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:35.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:35.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:35.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:35.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:35.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:35.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:35.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:35.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:35.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:35.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:35.918 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:35.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:36.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:36.033 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:36 smithi153 bash[59565]: cluster 2023-10-09T04:48:34.705484+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 708 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:48:36.053 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:36.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:36.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 21366 21350 148.231 116.5 0.00850692 0.00674046 2023-10-09T04:48:36.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:36 smithi082 bash[77063]: cluster 2023-10-09T04:48:34.705484+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 708 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:48:36.281 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:36 smithi082 bash[79733]: cluster 2023-10-09T04:48:34.705484+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 708 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:48:36.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:36.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:36.447 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:36.549 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:36.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:36.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:36.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:36.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:36.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:36.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:37.094 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:37.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:37.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:37.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 22978 22962 143.482 100.75 0.0052607 0.0069623 2023-10-09T04:48:37.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:37.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:37.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:37.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:37.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:37.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:37.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:37.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:37.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:37.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:37.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:37.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:37.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:37.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:38.017 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:38.120 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:38.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:38.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 16 24413 24397 138.592 89.6875 0.0120245 0.0072088 2023-10-09T04:48:38.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.262 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:38 smithi153 bash[59565]: cluster 2023-10-09T04:48:36.706460+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:48:38.278 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:38.382 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:38.390 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:38 smithi082 bash[77063]: cluster 2023-10-09T04:48:36.706460+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:48:38.391 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:38 smithi082 bash[79733]: cluster 2023-10-09T04:48:36.706460+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:48:38.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:38.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:38.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:38.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:38.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:38.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:38.910 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.931 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:38.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:39.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:39.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:39.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:39.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:39.202 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 26259 26243 136.656 115.375 0.00849193 0.00731063 2023-10-09T04:48:39.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:39.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:39.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:39.459 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:39.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:39.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:39.588 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:39.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:39.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:39.821 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:39.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:39.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:39.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:39] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.33.4" 2023-10-09T04:48:39.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:39.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:40.088 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:40 smithi153 bash[59565]: cluster 2023-10-09T04:48:38.706847+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:48:40.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:40.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 27985 27969 134.442 107.875 0.00380226 0.0074325 2023-10-09T04:48:40.219 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.240 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:40.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.352 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:40 smithi082 bash[77063]: cluster 2023-10-09T04:48:38.706847+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:48:40.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:40 smithi082 bash[79733]: cluster 2023-10-09T04:48:38.706847+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:48:40.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:40.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.499 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:40.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:40.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:40.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:40.891 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:40.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:40.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:40.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:41.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:41.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:41.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.151 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:41.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 29634 29618 132.2 103.062 0.00857665 0.00755843 2023-10-09T04:48:41.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:41.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:41.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:41.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:41.518 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:41.542 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:41.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:41.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:41.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:41.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:41.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:42.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:42.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:42.169 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:42 smithi153 bash[59565]: cluster 2023-10-09T04:48:40.707542+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-09T04:48:42.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 16 31335 31319 130.474 106.312 0.00647805 0.0076573 2023-10-09T04:48:42.208 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:42.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:42 smithi082 bash[77063]: cluster 2023-10-09T04:48:40.707542+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-09T04:48:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:42 smithi082 bash[79733]: cluster 2023-10-09T04:48:40.707542+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-09T04:48:42.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:42.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:42.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:42.482 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:42.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:42.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:42.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:42.879 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:42.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:42.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:43.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:43.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:43.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:43.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 32935 32919 128.569 100 0.00525861 0.00777133 2023-10-09T04:48:43.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:43.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:43.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:43.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:43.499 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:43.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:43.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:43.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:43.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.793 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:43.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:43.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:43.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:43.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:44.027 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:44.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:44.165 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:44.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:44.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 15 34428 34413 126.498 93.375 0.0109949 0.00789884 2023-10-09T04:48:44.268 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:44 smithi082 bash[77063]: cluster 2023-10-09T04:48:42.708186+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:48:44.268 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:44 smithi082 bash[79733]: cluster 2023-10-09T04:48:42.708186+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:48:44.297 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.339 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:44.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:44 smithi153 bash[59565]: cluster 2023-10-09T04:48:42.708186+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:48:44.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:44.577 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:44.610 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:44.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.742 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:44.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:44.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:44.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:44.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:45.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:45.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:45.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:45 smithi153 bash[59565]: audit 2023-10-09T04:48:44.290226+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:45.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 36033 36017 125.039 100.25 0.00558824 0.00799173 2023-10-09T04:48:45.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:45.251 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:45 smithi082 bash[77063]: audit 2023-10-09T04:48:44.290226+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:45.251 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:45 smithi082 bash[79733]: audit 2023-10-09T04:48:44.290226+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:45.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:45.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:45.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:45.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:45.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:45.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:45.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:45.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:45.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:45.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:45.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:45.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:45.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:45.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:46.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:46.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:46.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:46.177 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:46 smithi082 bash[77063]: cluster 2023-10-09T04:48:44.709657+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:48:46.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:46 smithi082 bash[79733]: cluster 2023-10-09T04:48:44.709657+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:48:46.192 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:46.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 16 37269 37253 122.524 77.25 0.00923967 0.00815633 2023-10-09T04:48:46.296 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.306 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:46 smithi153 bash[59565]: cluster 2023-10-09T04:48:44.709657+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:48:46.324 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:46.430 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:46.567 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:46.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:46.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:46.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:46.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:46.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:46.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:46.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:47.084 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:47.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:47.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:48:47.202654+0000 min lat: 0.00135091 max lat: 0.247697 avg lat: 0.00828951 2023-10-09T04:48:47.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:48:47.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 16 38597 38581 120.547 83 0.00645157 0.00828951 2023-10-09T04:48:47.217 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:47.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:47.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:47.374 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:47.476 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:47.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:47.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:47.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:47.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:47.763 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:47.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:47.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:47.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:48.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:48.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:48.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:48.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:48.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:48.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 39949 39933 118.831 84.5 0.0069904 0.00840894 2023-10-09T04:48:48.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:48 smithi082 bash[77063]: cluster 2023-10-09T04:48:46.710507+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T04:48:48.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:48 smithi082 bash[79733]: cluster 2023-10-09T04:48:46.710507+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T04:48:48.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:48.305 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:48.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:48 smithi153 bash[59565]: cluster 2023-10-09T04:48:46.710507+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T04:48:48.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:48.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:48.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:48.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:48.703 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:48.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:48.807 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:48.833 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:48.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:48.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:48.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:48.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:49.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.100 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:49.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 41329 41313 117.349 86.25 0.00975562 0.00851545 2023-10-09T04:48:49.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:49.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:49.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:49.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:49.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:49.730 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:49] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-10-09T04:48:49.759 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:49.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:49.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:50.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:50.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:50.133 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:50.143 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:50 smithi153 bash[59565]: cluster 2023-10-09T04:48:48.711040+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:50.159 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:50.203 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 16 42640 42624 115.809 81.9375 0.0143929 0.00862978 2023-10-09T04:48:50.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:50.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:50.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:50.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:50 smithi082 bash[77063]: cluster 2023-10-09T04:48:48.711040+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:50.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:50 smithi082 bash[79733]: cluster 2023-10-09T04:48:48.711040+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:50.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:50.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:50.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:50.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:50.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:50.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:50.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:50.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:50.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:51.053 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:51.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:51.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:51.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 43845 43829 114.122 75.3125 0.00561935 0.00875682 2023-10-09T04:48:51.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:51.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:51.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:51.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:51.480 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:51.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:51.614 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:51.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:51.717 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:51.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:51.853 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:51.887 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:51.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:51.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:52.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:52.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:52 smithi082 bash[77063]: cluster 2023-10-09T04:48:50.712121+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:48:52.139 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:52 smithi082 bash[79733]: cluster 2023-10-09T04:48:50.712121+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:48:52.140 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:48:52.163 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:52.180 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:48:52.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 45213 45197 112.977 85.5 0.0137774 0.00884625 2023-10-09T04:48:52.223 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:48:52.261 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:48:52.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.301 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:48:52.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:52.348 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:48:52.393 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:48:52.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:52.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:52 smithi153 bash[59565]: cluster 2023-10-09T04:48:50.712121+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:48:52.446 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:48:52.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:48:52.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:52.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:52.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:52.753 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:48:52.753 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:48:52.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:52.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:52.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:52.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:52.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:48:53.066 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:48:53.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:53.124 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:53.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 46327 46311 111.309 69.625 0.0187698 0.00897841 2023-10-09T04:48:53.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:53.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:53 smithi082 bash[77063]: audit 2023-10-09T04:48:52.104048+0000 mon.a (mon.0) 1528 : audit 0 from='client.? 172.21.15.82:0/1100834406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:53.367 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:53 smithi082 bash[79733]: audit 2023-10-09T04:48:52.104048+0000 mon.a (mon.0) 1528 : audit 0 from='client.? 172.21.15.82:0/1100834406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:53.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:53 smithi153 bash[59565]: audit 2023-10-09T04:48:52.104048+0000 mon.a (mon.0) 1528 : audit 0 from='client.? 172.21.15.82:0/1100834406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:53.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:53.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:53.634 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:53.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:53.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:53.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:53.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:53.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:53.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:54.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:54.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:54.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:54.086 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:48:54.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:54.165 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:54 smithi153 bash[59565]: cluster 2023-10-09T04:48:52.712657+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:48:54.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:54 smithi153 bash[59565]: audit 2023-10-09T04:48:53.035813+0000 mgr.y (mgr.34107) 1045 : audit 0 from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:54.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:54 smithi153 bash[59565]: audit 2023-10-09T04:48:53.139131+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.82:0/3211509148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.166 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:54 smithi153 bash[59565]: audit 2023-10-09T04:48:53.139763+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 47663 47647 110.279 83.5 0.0188532 0.00906153 2023-10-09T04:48:54.207 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:54.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.340 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[77063]: cluster 2023-10-09T04:48:52.712657+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[77063]: audit 2023-10-09T04:48:53.035813+0000 mgr.y (mgr.34107) 1045 : audit 0 from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[77063]: audit 2023-10-09T04:48:53.139131+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.82:0/3211509148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[77063]: audit 2023-10-09T04:48:53.139763+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[79733]: cluster 2023-10-09T04:48:52.712657+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[79733]: audit 2023-10-09T04:48:53.035813+0000 mgr.y (mgr.34107) 1045 : audit 0 from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[79733]: audit 2023-10-09T04:48:53.139131+0000 mon.c (mon.2) 115 : audit 1 from='client.? 172.21.15.82:0/3211509148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.341 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:54 smithi082 bash[79733]: audit 2023-10-09T04:48:53.139763+0000 mon.a (mon.0) 1529 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:54.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:54.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:54.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:54.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:54.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:54.601 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:54.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:54.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:54.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:54.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:54.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:54.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:55.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:55.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:55.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:55.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:55.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 15 49111 49096 109.575 90.5625 0.0118216 0.00912024 2023-10-09T04:48:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.068823+0000 mon.a (mon.0) 1530 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.074969+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.077049+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: audit 2023-10-09T04:48:54.077409+0000 mon.a (mon.0) 1531 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.077808+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.081688+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.083717+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.085674+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.068823+0000 mon.a (mon.0) 1530 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.074969+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.077049+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: audit 2023-10-09T04:48:54.077409+0000 mon.a (mon.0) 1531 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.077808+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.081688+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.083717+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:55 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.085674+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:55.283 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:55.386 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:55.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.068823+0000 mon.a (mon.0) 1530 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:55.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.074969+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.077049+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: audit 2023-10-09T04:48:54.077409+0000 mon.a (mon.0) 1531 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.077808+0000 mon.a (mon.0) 1532 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.081688+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.083717+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.396 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:55 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.085674+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T04:48:55.414 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:55.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:55.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:55.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:55.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:55.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:55.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:55.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:55.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:55.953 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:55.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:56.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:56.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:56.099 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:56 smithi082 bash[77063]: cluster 2023-10-09T04:48:54.713180+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:56.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:56.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 50463 50447 108.708 84.4375 0.0159713 0.00919234 2023-10-09T04:48:56.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:56.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:56.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:56.397 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:56 smithi082 bash[79733]: cluster 2023-10-09T04:48:54.713180+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:56.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:56 smithi153 bash[59565]: cluster 2023-10-09T04:48:54.713180+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:48:56.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:48:56.492 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1114,"stamp":"2023-10-09T04:48:52.712340+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2780501939,"num_objects":42382,"num_object_clones":0,"num_object_copies":127146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42382,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":43460,"num_write_kb":2725815,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8398292,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43524,"ondisk_log_size":43524,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12062128,"kb_used_data":8053296,"kb_used_omap":47,"kb_used_meta":4008720,"kb_avail":737899088,"statfs":{"total":767960285184,"available":755608666112,"internally_reserved":0,"allocated":8246575104,"data_stored":8240246764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":4104929693},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,1,4,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1064697856,"num_objects":16246,"num_object_clones":0,"num_object_copies":48738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16246,"num_write_kb":1039744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005318"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":249,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044782+0000","last_change":"2023-10-09T04:44:51.896564+0000","last_active":"2023-10-09T04:48:03.044782+0000","last_peered":"2023-10-09T04:48:03.044782+0000","last_clean":"2023-10-09T04:48:03.044782+0000","last_became_active":"2023-10-09T04:44:51.896195+0000","last_became_peered":"2023-10-09T04:44:51.896195+0000","last_unstale":"2023-10-09T04:48:03.044782+0000","last_undegraded":"2023-10-09T04:48:03.044782+0000","last_fullsized":"2023-10-09T04:48:03.044782+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":208,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.011641+0000","last_change":"2023-10-09T04:44:51.891799+0000","last_active":"2023-10-09T04:48:04.011641+0000","last_peered":"2023-10-09T04:48:04.011641+0000","last_clean":"2023-10-09T04:48:04.011641+0000","last_became_active":"2023-10-09T04:44:51.891479+0000","last_became_peered":"2023-10-09T04:44:51.891479+0000","last_unstale":"2023-10-09T04:48:04.011641+0000","last_undegraded":"2023-10-09T04:48:04.011641+0000","last_fullsized":"2023-10-09T04:48:04.011641+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:57.730414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":257,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442439+0000","last_change":"2023-10-09T04:43:53.989477+0000","last_active":"2023-10-09T04:48:03.442439+0000","last_peered":"2023-10-09T04:48:03.442439+0000","last_clean":"2023-10-09T04:48:03.442439+0000","last_became_active":"2023-10-09T04:43:53.988910+0000","last_became_peered":"2023-10-09T04:43:53.988910+0000","last_unstale":"2023-10-09T04:48:03.442439+0000","last_undegraded":"2023-10-09T04:48:03.442439+0000","last_fullsized":"2023-10-09T04:48:03.442439+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:35.144801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":231,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.865907+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:48:24.865907+0000","last_peered":"2023-10-09T04:48:24.865907+0000","last_clean":"2023-10-09T04:48:24.865907+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:48:24.865907+0000","last_undegraded":"2023-10-09T04:48:24.865907+0000","last_fullsized":"2023-10-09T04:48:24.865907+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":234,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:19.277591+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:48:19.277591+0000","last_peered":"2023-10-09T04:48:19.277591+0000","last_clean":"2023-10-09T04:48:19.277591+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:48:19.277591+0000","last_undegraded":"2023-10-09T04:48:19.277591+0000","last_fullsized":"2023-10-09T04:48:19.277591+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.046348+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:48:03.046348+0000","last_peered":"2023-10-09T04:48:03.046348+0000","last_clean":"2023-10-09T04:48:03.046348+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:48:03.046348+0000","last_undegraded":"2023-10-09T04:48:03.046348+0000","last_fullsized":"2023-10-09T04:48:03.046348+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":225,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039688+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:48:03.039688+0000","last_peered":"2023-10-09T04:48:03.039688+0000","last_clean":"2023-10-09T04:48:03.039688+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:48:03.039688+0000","last_undegraded":"2023-10-09T04:48:03.039688+0000","last_fullsized":"2023-10-09T04:48:03.039688+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":174,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.101433+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:48:24.101433+0000","last_peered":"2023-10-09T04:48:24.101433+0000","last_clean":"2023-10-09T04:48:24.101433+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:48:24.101433+0000","last_undegraded":"2023-10-09T04:48:24.101433+0000","last_fullsized":"2023-10-09T04:48:24.101433+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":194,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015143+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:48:04.015143+0000","last_peered":"2023-10-09T04:48:04.015143+0000","last_clean":"2023-10-09T04:48:04.015143+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:48:04.015143+0000","last_undegraded":"2023-10-09T04:48:04.015143+0000","last_fullsized":"2023-10-09T04:48:04.015143+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015180+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:48:04.015180+0000","last_peered":"2023-10-09T04:48:04.015180+0000","last_clean":"2023-10-09T04:48:04.015180+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:48:04.015180+0000","last_undegraded":"2023-10-09T04:48:04.015180+0000","last_fullsized":"2023-10-09T04:48:04.015180+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.016781+0000","last_change":"2023-10-09T04:40:47.996370+0000","last_active":"2023-10-09T04:48:04.016781+0000","last_peered":"2023-10-09T04:48:04.016781+0000","last_clean":"2023-10-09T04:48:04.016781+0000","last_became_active":"2023-10-09T04:40:47.817084+0000","last_became_peered":"2023-10-09T04:40:47.817084+0000","last_unstale":"2023-10-09T04:48:04.016781+0000","last_undegraded":"2023-10-09T04:48:04.016781+0000","last_fullsized":"2023-10-09T04:48:04.016781+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.319953+0000","last_change":"2023-10-09T04:44:51.903464+0000","last_active":"2023-10-09T04:48:24.319953+0000","last_peered":"2023-10-09T04:48:24.319953+0000","last_clean":"2023-10-09T04:48:24.319953+0000","last_became_active":"2023-10-09T04:44:51.903078+0000","last_became_peered":"2023-10-09T04:44:51.903078+0000","last_unstale":"2023-10-09T04:48:24.319953+0000","last_undegraded":"2023-10-09T04:48:24.319953+0000","last_fullsized":"2023-10-09T04:48:24.319953+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:57.992171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":278,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045530+0000","last_change":"2023-10-09T04:40:47.863039+0000","last_active":"2023-10-09T04:48:03.045530+0000","last_peered":"2023-10-09T04:48:03.045530+0000","last_clean":"2023-10-09T04:48:03.045530+0000","last_became_active":"2023-10-09T04:40:47.819279+0000","last_became_peered":"2023-10-09T04:40:47.819279+0000","last_unstale":"2023-10-09T04:48:03.045530+0000","last_undegraded":"2023-10-09T04:48:03.045530+0000","last_fullsized":"2023-10-09T04:48:03.045530+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045399+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:48:03.045399+0000","last_peered":"2023-10-09T04:48:03.045399+0000","last_clean":"2023-10-09T04:48:03.045399+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:48:03.045399+0000","last_undegraded":"2023-10-09T04:48:03.045399+0000","last_fullsized":"2023-10-09T04:48:03.045399+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":89,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.013844+0000","last_change":"2023-10-09T04:44:51.899516+0000","last_active":"2023-10-09T04:48:04.013844+0000","last_peered":"2023-10-09T04:48:04.013844+0000","last_clean":"2023-10-09T04:48:04.013844+0000","last_became_active":"2023-10-09T04:44:51.899092+0000","last_became_peered":"2023-10-09T04:44:51.899092+0000","last_unstale":"2023-10-09T04:48:04.013844+0000","last_undegraded":"2023-10-09T04:48:04.013844+0000","last_fullsized":"2023-10-09T04:48:04.013844+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.443045+0000","last_change":"2023-10-09T04:40:47.816775+0000","last_active":"2023-10-09T04:48:03.443045+0000","last_peered":"2023-10-09T04:48:03.443045+0000","last_clean":"2023-10-09T04:48:03.443045+0000","last_became_active":"2023-10-09T04:40:47.816424+0000","last_became_peered":"2023-10-09T04:40:47.816424+0000","last_unstale":"2023-10-09T04:48:03.443045+0000","last_undegraded":"2023-10-09T04:48:03.443045+0000","last_fullsized":"2023-10-09T04:48:03.443045+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:52.923977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":229,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.275314+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:48:04.275314+0000","last_peered":"2023-10-09T04:48:04.275314+0000","last_clean":"2023-10-09T04:48:04.275314+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:48:04.275314+0000","last_undegraded":"2023-10-09T04:48:04.275314+0000","last_fullsized":"2023-10-09T04:48:04.275314+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":203,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015342+0000","last_change":"2023-10-09T04:40:47.820893+0000","last_active":"2023-10-09T04:48:04.015342+0000","last_peered":"2023-10-09T04:48:04.015342+0000","last_clean":"2023-10-09T04:48:04.015342+0000","last_became_active":"2023-10-09T04:40:47.820570+0000","last_became_peered":"2023-10-09T04:40:47.820570+0000","last_unstale":"2023-10-09T04:48:04.015342+0000","last_undegraded":"2023-10-09T04:48:04.015342+0000","last_fullsized":"2023-10-09T04:48:04.015342+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":68,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014978+0000","last_change":"2023-10-09T04:40:47.823962+0000","last_active":"2023-10-09T04:48:04.014978+0000","last_peered":"2023-10-09T04:48:04.014978+0000","last_clean":"2023-10-09T04:48:04.014978+0000","last_became_active":"2023-10-09T04:40:47.823509+0000","last_became_peered":"2023-10-09T04:40:47.823509+0000","last_unstale":"2023-10-09T04:48:04.014978+0000","last_undegraded":"2023-10-09T04:48:04.014978+0000","last_fullsized":"2023-10-09T04:48:04.014978+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":228,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.040544+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:48:03.040544+0000","last_peered":"2023-10-09T04:48:03.040544+0000","last_clean":"2023-10-09T04:48:03.040544+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:48:03.040544+0000","last_undegraded":"2023-10-09T04:48:03.040544+0000","last_fullsized":"2023-10-09T04:48:03.040544+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":217,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045495+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:48:03.045495+0000","last_peered":"2023-10-09T04:48:03.045495+0000","last_clean":"2023-10-09T04:48:03.045495+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:48:03.045495+0000","last_undegraded":"2023-10-09T04:48:03.045495+0000","last_fullsized":"2023-10-09T04:48:03.045495+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.102636+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:48:24.102636+0000","last_peered":"2023-10-09T04:48:24.102636+0000","last_clean":"2023-10-09T04:48:24.102636+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:48:24.102636+0000","last_undegraded":"2023-10-09T04:48:24.102636+0000","last_fullsized":"2023-10-09T04:48:24.102636+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"253'2599","reported_seq":2611,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.273173+0000","last_change":"2023-10-09T04:48:02.044407+0000","last_active":"2023-10-09T04:48:49.273173+0000","last_peered":"2023-10-09T04:48:49.273173+0000","last_clean":"2023-10-09T04:48:49.273173+0000","last_became_active":"2023-10-09T04:48:02.043959+0000","last_became_peered":"2023-10-09T04:48:02.043959+0000","last_unstale":"2023-10-09T04:48:49.273173+0000","last_undegraded":"2023-10-09T04:48:49.273173+0000","last_fullsized":"2023-10-09T04:48:49.273173+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2599,"log_dups_size":0,"ondisk_log_size":2599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:02.677050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170328064,"num_objects":2599,"num_object_clones":0,"num_object_copies":7797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2599,"num_write_kb":166336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.016892+0000","last_change":"2023-10-09T04:40:48.230655+0000","last_active":"2023-10-09T04:48:04.016892+0000","last_peered":"2023-10-09T04:48:04.016892+0000","last_clean":"2023-10-09T04:48:04.016892+0000","last_became_active":"2023-10-09T04:40:47.830826+0000","last_became_peered":"2023-10-09T04:40:47.830826+0000","last_unstale":"2023-10-09T04:48:04.016892+0000","last_undegraded":"2023-10-09T04:48:04.016892+0000","last_fullsized":"2023-10-09T04:48:04.016892+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":185,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.016045+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:48:04.016045+0000","last_peered":"2023-10-09T04:48:04.016045+0000","last_clean":"2023-10-09T04:48:04.016045+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:48:04.016045+0000","last_undegraded":"2023-10-09T04:48:04.016045+0000","last_fullsized":"2023-10-09T04:48:04.016045+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"253'2616","reported_seq":2628,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.718682+0000","last_change":"2023-10-09T04:48:02.139233+0000","last_active":"2023-10-09T04:48:49.718682+0000","last_peered":"2023-10-09T04:48:49.718682+0000","last_clean":"2023-10-09T04:48:49.718682+0000","last_became_active":"2023-10-09T04:48:02.138632+0000","last_became_peered":"2023-10-09T04:48:02.138632+0000","last_unstale":"2023-10-09T04:48:49.718682+0000","last_undegraded":"2023-10-09T04:48:49.718682+0000","last_fullsized":"2023-10-09T04:48:49.718682+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2616,"log_dups_size":0,"ondisk_log_size":2616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:52.944610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171442176,"num_objects":2616,"num_object_clones":0,"num_object_copies":7848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2616,"num_write_kb":167424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":85,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.017053+0000","last_change":"2023-10-09T04:43:54.055892+0000","last_active":"2023-10-09T04:48:04.017053+0000","last_peered":"2023-10-09T04:48:04.017053+0000","last_clean":"2023-10-09T04:48:04.017053+0000","last_became_active":"2023-10-09T04:43:53.987210+0000","last_became_peered":"2023-10-09T04:43:53.987210+0000","last_unstale":"2023-10-09T04:48:04.017053+0000","last_undegraded":"2023-10-09T04:48:04.017053+0000","last_fullsized":"2023-10-09T04:48:04.017053+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044715+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:48:03.044715+0000","last_peered":"2023-10-09T04:48:03.044715+0000","last_clean":"2023-10-09T04:48:03.044715+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:48:03.044715+0000","last_undegraded":"2023-10-09T04:48:03.044715+0000","last_fullsized":"2023-10-09T04:48:03.044715+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"253'2709","reported_seq":2720,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.576619+0000","last_change":"2023-10-09T04:48:02.157308+0000","last_active":"2023-10-09T04:48:50.576619+0000","last_peered":"2023-10-09T04:48:50.576619+0000","last_clean":"2023-10-09T04:48:50.576619+0000","last_became_active":"2023-10-09T04:48:02.156513+0000","last_became_peered":"2023-10-09T04:48:02.156513+0000","last_unstale":"2023-10-09T04:48:50.576619+0000","last_undegraded":"2023-10-09T04:48:50.576619+0000","last_fullsized":"2023-10-09T04:48:50.576619+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2709,"log_dups_size":0,"ondisk_log_size":2709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177537024,"num_objects":2709,"num_object_clones":0,"num_object_copies":8127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2709,"num_write_kb":173376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":72,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.016817+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:48:04.016817+0000","last_peered":"2023-10-09T04:48:04.016817+0000","last_clean":"2023-10-09T04:48:04.016817+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:48:04.016817+0000","last_undegraded":"2023-10-09T04:48:04.016817+0000","last_fullsized":"2023-10-09T04:48:04.016817+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":237,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045956+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:48:03.045956+0000","last_peered":"2023-10-09T04:48:03.045956+0000","last_clean":"2023-10-09T04:48:03.045956+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:48:03.045956+0000","last_undegraded":"2023-10-09T04:48:03.045956+0000","last_fullsized":"2023-10-09T04:48:03.045956+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"253'2705","reported_seq":2716,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.563980+0000","last_change":"2023-10-09T04:48:02.045217+0000","last_active":"2023-10-09T04:48:50.563980+0000","last_peered":"2023-10-09T04:48:50.563980+0000","last_clean":"2023-10-09T04:48:50.563980+0000","last_became_active":"2023-10-09T04:48:02.044927+0000","last_became_peered":"2023-10-09T04:48:02.044927+0000","last_unstale":"2023-10-09T04:48:50.563980+0000","last_undegraded":"2023-10-09T04:48:50.563980+0000","last_fullsized":"2023-10-09T04:48:50.563980+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2705,"log_dups_size":0,"ondisk_log_size":2705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2705,"num_write_kb":173120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014717+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:48:04.014717+0000","last_peered":"2023-10-09T04:48:04.014717+0000","last_clean":"2023-10-09T04:48:04.014717+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:48:04.014717+0000","last_undegraded":"2023-10-09T04:48:04.014717+0000","last_fullsized":"2023-10-09T04:48:04.014717+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":239,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039081+0000","last_change":"2023-10-09T04:44:51.892263+0000","last_active":"2023-10-09T04:48:03.039081+0000","last_peered":"2023-10-09T04:48:03.039081+0000","last_clean":"2023-10-09T04:48:03.039081+0000","last_became_active":"2023-10-09T04:44:51.891917+0000","last_became_peered":"2023-10-09T04:44:51.891917+0000","last_unstale":"2023-10-09T04:48:03.039081+0000","last_undegraded":"2023-10-09T04:48:03.039081+0000","last_fullsized":"2023-10-09T04:48:03.039081+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.796713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"253'2481","reported_seq":2492,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:48.494420+0000","last_change":"2023-10-09T04:48:02.156998+0000","last_active":"2023-10-09T04:48:48.494420+0000","last_peered":"2023-10-09T04:48:48.494420+0000","last_clean":"2023-10-09T04:48:48.494420+0000","last_became_active":"2023-10-09T04:48:02.156119+0000","last_became_peered":"2023-10-09T04:48:02.156119+0000","last_unstale":"2023-10-09T04:48:48.494420+0000","last_undegraded":"2023-10-09T04:48:48.494420+0000","last_fullsized":"2023-10-09T04:48:48.494420+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2481,"log_dups_size":0,"ondisk_log_size":2481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2481,"num_write_kb":158784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":174,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039324+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:48:03.039324+0000","last_peered":"2023-10-09T04:48:03.039324+0000","last_clean":"2023-10-09T04:48:03.039324+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:48:03.039324+0000","last_undegraded":"2023-10-09T04:48:03.039324+0000","last_fullsized":"2023-10-09T04:48:03.039324+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045708+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:48:03.045708+0000","last_peered":"2023-10-09T04:48:03.045708+0000","last_clean":"2023-10-09T04:48:03.045708+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:48:03.045708+0000","last_undegraded":"2023-10-09T04:48:03.045708+0000","last_fullsized":"2023-10-09T04:48:03.045708+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"253'2645","reported_seq":2656,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.723757+0000","last_change":"2023-10-09T04:48:02.036963+0000","last_active":"2023-10-09T04:48:49.723757+0000","last_peered":"2023-10-09T04:48:49.723757+0000","last_clean":"2023-10-09T04:48:49.723757+0000","last_became_active":"2023-10-09T04:48:02.036549+0000","last_became_peered":"2023-10-09T04:48:02.036549+0000","last_unstale":"2023-10-09T04:48:49.723757+0000","last_undegraded":"2023-10-09T04:48:49.723757+0000","last_fullsized":"2023-10-09T04:48:49.723757+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2645,"log_dups_size":0,"ondisk_log_size":2645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.195147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173342720,"num_objects":2645,"num_object_clones":0,"num_object_copies":7935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2645,"num_write_kb":169280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":220,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045662+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:48:03.045662+0000","last_peered":"2023-10-09T04:48:03.045662+0000","last_clean":"2023-10-09T04:48:03.045662+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:48:03.045662+0000","last_undegraded":"2023-10-09T04:48:03.045662+0000","last_fullsized":"2023-10-09T04:48:03.045662+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":46,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045176+0000","last_change":"2023-10-09T04:44:53.333649+0000","last_active":"2023-10-09T04:48:03.045176+0000","last_peered":"2023-10-09T04:48:03.045176+0000","last_clean":"2023-10-09T04:48:03.045176+0000","last_became_active":"2023-10-09T04:44:53.332464+0000","last_became_peered":"2023-10-09T04:44:53.332464+0000","last_unstale":"2023-10-09T04:48:03.045176+0000","last_undegraded":"2023-10-09T04:48:03.045176+0000","last_fullsized":"2023-10-09T04:48:03.045176+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"253'2677","reported_seq":2688,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.835044+0000","last_change":"2023-10-09T04:48:02.140445+0000","last_active":"2023-10-09T04:48:50.835044+0000","last_peered":"2023-10-09T04:48:50.835044+0000","last_clean":"2023-10-09T04:48:50.835044+0000","last_became_active":"2023-10-09T04:48:02.139406+0000","last_became_peered":"2023-10-09T04:48:02.139406+0000","last_unstale":"2023-10-09T04:48:50.835044+0000","last_undegraded":"2023-10-09T04:48:50.835044+0000","last_fullsized":"2023-10-09T04:48:50.835044+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2677,"log_dups_size":0,"ondisk_log_size":2677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175439872,"num_objects":2677,"num_object_clones":0,"num_object_copies":8031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2677,"num_write_kb":171328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":46,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044547+0000","last_change":"2023-10-09T04:44:53.102191+0000","last_active":"2023-10-09T04:48:03.044547+0000","last_peered":"2023-10-09T04:48:03.044547+0000","last_clean":"2023-10-09T04:48:03.044547+0000","last_became_active":"2023-10-09T04:44:53.101206+0000","last_became_peered":"2023-10-09T04:44:53.101206+0000","last_unstale":"2023-10-09T04:48:03.044547+0000","last_undegraded":"2023-10-09T04:48:03.044547+0000","last_fullsized":"2023-10-09T04:48:03.044547+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.047393+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:48:03.047393+0000","last_peered":"2023-10-09T04:48:03.047393+0000","last_clean":"2023-10-09T04:48:03.047393+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:48:03.047393+0000","last_undegraded":"2023-10-09T04:48:03.047393+0000","last_fullsized":"2023-10-09T04:48:03.047393+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"253'2660","reported_seq":2672,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.718008+0000","last_change":"2023-10-09T04:48:02.139290+0000","last_active":"2023-10-09T04:48:49.718008+0000","last_peered":"2023-10-09T04:48:49.718008+0000","last_clean":"2023-10-09T04:48:49.718008+0000","last_became_active":"2023-10-09T04:48:02.138631+0000","last_became_peered":"2023-10-09T04:48:02.138631+0000","last_unstale":"2023-10-09T04:48:49.718008+0000","last_undegraded":"2023-10-09T04:48:49.718008+0000","last_fullsized":"2023-10-09T04:48:49.718008+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2660,"log_dups_size":0,"ondisk_log_size":2660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:32.990209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2660,"num_write_kb":170240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":229,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.010610+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:48:04.010610+0000","last_peered":"2023-10-09T04:48:04.010610+0000","last_clean":"2023-10-09T04:48:04.010610+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:48:04.010610+0000","last_undegraded":"2023-10-09T04:48:04.010610+0000","last_fullsized":"2023-10-09T04:48:04.010610+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":207,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.443524+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:48:03.443524+0000","last_peered":"2023-10-09T04:48:03.443524+0000","last_clean":"2023-10-09T04:48:03.443524+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:48:03.443524+0000","last_undegraded":"2023-10-09T04:48:03.443524+0000","last_fullsized":"2023-10-09T04:48:03.443524+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"253'2745","reported_seq":2756,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.563738+0000","last_change":"2023-10-09T04:48:02.156639+0000","last_active":"2023-10-09T04:48:50.563738+0000","last_peered":"2023-10-09T04:48:50.563738+0000","last_clean":"2023-10-09T04:48:50.563738+0000","last_became_active":"2023-10-09T04:48:02.155893+0000","last_became_peered":"2023-10-09T04:48:02.155893+0000","last_unstale":"2023-10-09T04:48:50.563738+0000","last_undegraded":"2023-10-09T04:48:50.563738+0000","last_fullsized":"2023-10-09T04:48:50.563738+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2745,"log_dups_size":0,"ondisk_log_size":2745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179896320,"num_objects":2745,"num_object_clones":0,"num_object_copies":8235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2745,"num_write_kb":175680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":707,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.443232+0000","last_change":"2023-10-09T04:41:04.902111+0000","last_active":"2023-10-09T04:48:03.443232+0000","last_peered":"2023-10-09T04:48:03.443232+0000","last_clean":"2023-10-09T04:48:03.443232+0000","last_became_active":"2023-10-09T04:41:04.901604+0000","last_became_peered":"2023-10-09T04:41:04.901604+0000","last_unstale":"2023-10-09T04:48:03.443232+0000","last_undegraded":"2023-10-09T04:48:03.443232+0000","last_fullsized":"2023-10-09T04:48:03.443232+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:55.374183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":216,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015549+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:48:04.015549+0000","last_peered":"2023-10-09T04:48:04.015549+0000","last_clean":"2023-10-09T04:48:04.015549+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:48:04.015549+0000","last_undegraded":"2023-10-09T04:48:04.015549+0000","last_fullsized":"2023-10-09T04:48:04.015549+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":226,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442781+0000","last_change":"2023-10-09T04:43:53.993014+0000","last_active":"2023-10-09T04:48:03.442781+0000","last_peered":"2023-10-09T04:48:03.442781+0000","last_clean":"2023-10-09T04:48:03.442781+0000","last_became_active":"2023-10-09T04:43:53.992702+0000","last_became_peered":"2023-10-09T04:43:53.992702+0000","last_unstale":"2023-10-09T04:48:03.442781+0000","last_undegraded":"2023-10-09T04:48:03.442781+0000","last_fullsized":"2023-10-09T04:48:03.442781+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:47.151693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"253'2624","reported_seq":2636,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.285150+0000","last_change":"2023-10-09T04:48:02.038776+0000","last_active":"2023-10-09T04:48:49.285150+0000","last_peered":"2023-10-09T04:48:49.285150+0000","last_clean":"2023-10-09T04:48:49.285150+0000","last_became_active":"2023-10-09T04:48:02.038418+0000","last_became_peered":"2023-10-09T04:48:02.038418+0000","last_unstale":"2023-10-09T04:48:49.285150+0000","last_undegraded":"2023-10-09T04:48:49.285150+0000","last_fullsized":"2023-10-09T04:48:49.285150+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2624,"log_dups_size":0,"ondisk_log_size":2624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:26.911517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171966464,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2624,"num_write_kb":167936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":715,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015262+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:48:04.015262+0000","last_peered":"2023-10-09T04:48:04.015262+0000","last_clean":"2023-10-09T04:48:04.015262+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:48:04.015262+0000","last_undegraded":"2023-10-09T04:48:04.015262+0000","last_fullsized":"2023-10-09T04:48:04.015262+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":233,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045282+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:48:03.045282+0000","last_peered":"2023-10-09T04:48:03.045282+0000","last_clean":"2023-10-09T04:48:03.045282+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:48:03.045282+0000","last_undegraded":"2023-10-09T04:48:03.045282+0000","last_fullsized":"2023-10-09T04:48:03.045282+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.443047+0000","last_change":"2023-10-09T04:44:51.896804+0000","last_active":"2023-10-09T04:48:03.443047+0000","last_peered":"2023-10-09T04:48:03.443047+0000","last_clean":"2023-10-09T04:48:03.443047+0000","last_became_active":"2023-10-09T04:44:51.896434+0000","last_became_peered":"2023-10-09T04:44:51.896434+0000","last_unstale":"2023-10-09T04:48:03.443047+0000","last_undegraded":"2023-10-09T04:48:03.443047+0000","last_fullsized":"2023-10-09T04:48:03.443047+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:39.726149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"253'2750","reported_seq":2760,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.574080+0000","last_change":"2023-10-09T04:48:02.046283+0000","last_active":"2023-10-09T04:48:50.574080+0000","last_peered":"2023-10-09T04:48:50.574080+0000","last_clean":"2023-10-09T04:48:50.574080+0000","last_became_active":"2023-10-09T04:48:02.045795+0000","last_became_peered":"2023-10-09T04:48:02.045795+0000","last_unstale":"2023-10-09T04:48:50.574080+0000","last_undegraded":"2023-10-09T04:48:50.574080+0000","last_fullsized":"2023-10-09T04:48:50.574080+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2750,"log_dups_size":0,"ondisk_log_size":2750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2750,"num_write_kb":176000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1330,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.010488+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:48:04.010488+0000","last_peered":"2023-10-09T04:48:04.010488+0000","last_clean":"2023-10-09T04:48:04.010488+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:48:04.010488+0000","last_undegraded":"2023-10-09T04:48:04.010488+0000","last_fullsized":"2023-10-09T04:48:04.010488+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.442582+0000","last_change":"2023-10-09T04:44:51.903679+0000","last_active":"2023-10-09T04:48:03.442582+0000","last_peered":"2023-10-09T04:48:03.442582+0000","last_clean":"2023-10-09T04:48:03.442582+0000","last_became_active":"2023-10-09T04:44:51.903243+0000","last_became_peered":"2023-10-09T04:44:51.903243+0000","last_unstale":"2023-10-09T04:48:03.442582+0000","last_undegraded":"2023-10-09T04:48:03.442582+0000","last_fullsized":"2023-10-09T04:48:03.442582+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:35.333537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.320921+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:48:24.320921+0000","last_peered":"2023-10-09T04:48:24.320921+0000","last_clean":"2023-10-09T04:48:24.320921+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:48:24.320921+0000","last_undegraded":"2023-10-09T04:48:24.320921+0000","last_fullsized":"2023-10-09T04:48:24.320921+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"253'2645","reported_seq":2657,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.275249+0000","last_change":"2023-10-09T04:48:02.139821+0000","last_active":"2023-10-09T04:48:49.275249+0000","last_peered":"2023-10-09T04:48:49.275249+0000","last_clean":"2023-10-09T04:48:49.275249+0000","last_became_active":"2023-10-09T04:48:02.139065+0000","last_became_peered":"2023-10-09T04:48:02.139065+0000","last_unstale":"2023-10-09T04:48:49.275249+0000","last_undegraded":"2023-10-09T04:48:49.275249+0000","last_fullsized":"2023-10-09T04:48:49.275249+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2645,"log_dups_size":0,"ondisk_log_size":2645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:29.788840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173342720,"num_objects":2645,"num_object_clones":0,"num_object_copies":7935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2645,"num_write_kb":169280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":778,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015613+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:48:04.015613+0000","last_peered":"2023-10-09T04:48:04.015613+0000","last_clean":"2023-10-09T04:48:04.015613+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:48:04.015613+0000","last_undegraded":"2023-10-09T04:48:04.015613+0000","last_fullsized":"2023-10-09T04:48:04.015613+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":474,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014345+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:48:04.014345+0000","last_peered":"2023-10-09T04:48:04.014345+0000","last_clean":"2023-10-09T04:48:04.014345+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:48:04.014345+0000","last_undegraded":"2023-10-09T04:48:04.014345+0000","last_fullsized":"2023-10-09T04:48:04.014345+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015037+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:48:04.015037+0000","last_peered":"2023-10-09T04:48:04.015037+0000","last_clean":"2023-10-09T04:48:04.015037+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:48:04.015037+0000","last_undegraded":"2023-10-09T04:48:04.015037+0000","last_fullsized":"2023-10-09T04:48:04.015037+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":187,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.046199+0000","last_change":"2023-10-09T04:40:47.825563+0000","last_active":"2023-10-09T04:48:03.046199+0000","last_peered":"2023-10-09T04:48:03.046199+0000","last_clean":"2023-10-09T04:48:03.046199+0000","last_became_active":"2023-10-09T04:40:47.824020+0000","last_became_peered":"2023-10-09T04:40:47.824020+0000","last_unstale":"2023-10-09T04:48:03.046199+0000","last_undegraded":"2023-10-09T04:48:03.046199+0000","last_fullsized":"2023-10-09T04:48:03.046199+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"253'2594","reported_seq":2604,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:49.293913+0000","last_change":"2023-10-09T04:48:02.042880+0000","last_active":"2023-10-09T04:48:49.293913+0000","last_peered":"2023-10-09T04:48:49.293913+0000","last_clean":"2023-10-09T04:48:49.293913+0000","last_became_active":"2023-10-09T04:48:02.042529+0000","last_became_peered":"2023-10-09T04:48:02.042529+0000","last_unstale":"2023-10-09T04:48:49.293913+0000","last_undegraded":"2023-10-09T04:48:49.293913+0000","last_fullsized":"2023-10-09T04:48:49.293913+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2594,"log_dups_size":0,"ondisk_log_size":2594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:32.083166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170000384,"num_objects":2594,"num_object_clones":0,"num_object_copies":7782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2594,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":640,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014262+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:48:04.014262+0000","last_peered":"2023-10-09T04:48:04.014262+0000","last_clean":"2023-10-09T04:48:04.014262+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:48:04.014262+0000","last_undegraded":"2023-10-09T04:48:04.014262+0000","last_fullsized":"2023-10-09T04:48:04.014262+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":259,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.038846+0000","last_change":"2023-10-09T04:43:53.972134+0000","last_active":"2023-10-09T04:48:03.038846+0000","last_peered":"2023-10-09T04:48:03.038846+0000","last_clean":"2023-10-09T04:48:03.038846+0000","last_became_active":"2023-10-09T04:43:53.971710+0000","last_became_peered":"2023-10-09T04:43:53.971710+0000","last_unstale":"2023-10-09T04:48:03.038846+0000","last_undegraded":"2023-10-09T04:48:03.038846+0000","last_fullsized":"2023-10-09T04:48:03.038846+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:56.047480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":242,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.278766+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:48:24.278766+0000","last_peered":"2023-10-09T04:48:24.278766+0000","last_clean":"2023-10-09T04:48:24.278766+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:48:24.278766+0000","last_undegraded":"2023-10-09T04:48:24.278766+0000","last_fullsized":"2023-10-09T04:48:24.278766+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"253'2709","reported_seq":2720,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.812163+0000","last_change":"2023-10-09T04:48:02.043240+0000","last_active":"2023-10-09T04:48:50.812163+0000","last_peered":"2023-10-09T04:48:50.812163+0000","last_clean":"2023-10-09T04:48:50.812163+0000","last_became_active":"2023-10-09T04:48:02.042897+0000","last_became_peered":"2023-10-09T04:48:02.042897+0000","last_unstale":"2023-10-09T04:48:50.812163+0000","last_undegraded":"2023-10-09T04:48:50.812163+0000","last_fullsized":"2023-10-09T04:48:50.812163+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2709,"log_dups_size":0,"ondisk_log_size":2709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177537024,"num_objects":2709,"num_object_clones":0,"num_object_copies":8127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2709,"num_write_kb":173376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":470,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.016891+0000","last_change":"2023-10-09T04:40:48.818879+0000","last_active":"2023-10-09T04:48:04.016891+0000","last_peered":"2023-10-09T04:48:04.016891+0000","last_clean":"2023-10-09T04:48:04.016891+0000","last_became_active":"2023-10-09T04:40:48.818073+0000","last_became_peered":"2023-10-09T04:40:48.818073+0000","last_unstale":"2023-10-09T04:48:04.016891+0000","last_undegraded":"2023-10-09T04:48:04.016891+0000","last_fullsized":"2023-10-09T04:48:04.016891+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.443191+0000","last_change":"2023-10-09T04:44:51.897117+0000","last_active":"2023-10-09T04:48:03.443191+0000","last_peered":"2023-10-09T04:48:03.443191+0000","last_clean":"2023-10-09T04:48:03.443191+0000","last_became_active":"2023-10-09T04:44:51.896672+0000","last_became_peered":"2023-10-09T04:44:51.896672+0000","last_unstale":"2023-10-09T04:48:03.443191+0000","last_undegraded":"2023-10-09T04:48:03.443191+0000","last_fullsized":"2023-10-09T04:48:03.443191+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:09.426260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039524+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:48:03.039524+0000","last_peered":"2023-10-09T04:48:03.039524+0000","last_clean":"2023-10-09T04:48:03.039524+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:48:03.039524+0000","last_undegraded":"2023-10-09T04:48:03.039524+0000","last_fullsized":"2023-10-09T04:48:03.039524+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"253'2610","reported_seq":2621,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.838745+0000","last_change":"2023-10-09T04:48:02.046557+0000","last_active":"2023-10-09T04:48:50.838745+0000","last_peered":"2023-10-09T04:48:50.838745+0000","last_clean":"2023-10-09T04:48:50.838745+0000","last_became_active":"2023-10-09T04:48:02.046207+0000","last_became_peered":"2023-10-09T04:48:02.046207+0000","last_unstale":"2023-10-09T04:48:50.838745+0000","last_undegraded":"2023-10-09T04:48:50.838745+0000","last_fullsized":"2023-10-09T04:48:50.838745+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2610,"log_dups_size":0,"ondisk_log_size":2610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2610,"num_write_kb":167040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.017073+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:48:04.017073+0000","last_peered":"2023-10-09T04:48:04.017073+0000","last_clean":"2023-10-09T04:48:04.017073+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:48:04.017073+0000","last_undegraded":"2023-10-09T04:48:04.017073+0000","last_fullsized":"2023-10-09T04:48:04.017073+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":499,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.320759+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:48:24.320759+0000","last_peered":"2023-10-09T04:48:24.320759+0000","last_clean":"2023-10-09T04:48:24.320759+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:48:24.320759+0000","last_undegraded":"2023-10-09T04:48:24.320759+0000","last_fullsized":"2023-10-09T04:48:24.320759+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014134+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:48:04.014134+0000","last_peered":"2023-10-09T04:48:04.014134+0000","last_clean":"2023-10-09T04:48:04.014134+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:48:04.014134+0000","last_undegraded":"2023-10-09T04:48:04.014134+0000","last_fullsized":"2023-10-09T04:48:04.014134+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"253'2585","reported_seq":2596,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.112249+0000","last_change":"2023-10-09T04:48:02.156388+0000","last_active":"2023-10-09T04:48:50.112249+0000","last_peered":"2023-10-09T04:48:50.112249+0000","last_clean":"2023-10-09T04:48:50.112249+0000","last_became_active":"2023-10-09T04:48:02.155602+0000","last_became_peered":"2023-10-09T04:48:02.155602+0000","last_unstale":"2023-10-09T04:48:50.112249+0000","last_undegraded":"2023-10-09T04:48:50.112249+0000","last_fullsized":"2023-10-09T04:48:50.112249+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":2585,"log_dups_size":0,"ondisk_log_size":2585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2585,"num_write_kb":165440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":333,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.046243+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:48:03.046243+0000","last_peered":"2023-10-09T04:48:03.046243+0000","last_clean":"2023-10-09T04:48:03.046243+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:48:03.046243+0000","last_undegraded":"2023-10-09T04:48:03.046243+0000","last_fullsized":"2023-10-09T04:48:03.046243+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":216,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.278797+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:48:24.278797+0000","last_peered":"2023-10-09T04:48:24.278797+0000","last_clean":"2023-10-09T04:48:24.278797+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:48:24.278797+0000","last_undegraded":"2023-10-09T04:48:24.278797+0000","last_fullsized":"2023-10-09T04:48:24.278797+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":35,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044581+0000","last_change":"2023-10-09T04:43:53.980738+0000","last_active":"2023-10-09T04:48:03.044581+0000","last_peered":"2023-10-09T04:48:03.044581+0000","last_clean":"2023-10-09T04:48:03.044581+0000","last_became_active":"2023-10-09T04:43:53.980299+0000","last_became_peered":"2023-10-09T04:43:53.980299+0000","last_unstale":"2023-10-09T04:48:03.044581+0000","last_undegraded":"2023-10-09T04:48:03.044581+0000","last_fullsized":"2023-10-09T04:48:03.044581+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":217,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039431+0000","last_change":"2023-10-09T04:40:47.823824+0000","last_active":"2023-10-09T04:48:03.039431+0000","last_peered":"2023-10-09T04:48:03.039431+0000","last_clean":"2023-10-09T04:48:03.039431+0000","last_became_active":"2023-10-09T04:40:47.817680+0000","last_became_peered":"2023-10-09T04:40:47.817680+0000","last_unstale":"2023-10-09T04:48:03.039431+0000","last_undegraded":"2023-10-09T04:48:03.039431+0000","last_fullsized":"2023-10-09T04:48:03.039431+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:07.850102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":197,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.012453+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:48:04.012453+0000","last_peered":"2023-10-09T04:48:04.012453+0000","last_clean":"2023-10-09T04:48:04.012453+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:48:04.012453+0000","last_undegraded":"2023-10-09T04:48:04.012453+0000","last_fullsized":"2023-10-09T04:48:04.012453+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":178,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.014949+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:48:04.014949+0000","last_peered":"2023-10-09T04:48:04.014949+0000","last_clean":"2023-10-09T04:48:04.014949+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:48:04.014949+0000","last_undegraded":"2023-10-09T04:48:04.014949+0000","last_fullsized":"2023-10-09T04:48:04.014949+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.045060+0000","last_change":"2023-10-09T04:43:53.974915+0000","last_active":"2023-10-09T04:48:03.045060+0000","last_peered":"2023-10-09T04:48:03.045060+0000","last_clean":"2023-10-09T04:48:03.045060+0000","last_became_active":"2023-10-09T04:43:53.974611+0000","last_became_peered":"2023-10-09T04:43:53.974611+0000","last_unstale":"2023-10-09T04:48:03.045060+0000","last_undegraded":"2023-10-09T04:48:03.045060+0000","last_fullsized":"2023-10-09T04:48:03.045060+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":171,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.015780+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:48:04.015780+0000","last_peered":"2023-10-09T04:48:04.015780+0000","last_clean":"2023-10-09T04:48:04.015780+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:48:04.015780+0000","last_undegraded":"2023-10-09T04:48:04.015780+0000","last_fullsized":"2023-10-09T04:48:04.015780+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044885+0000","last_change":"2023-10-09T04:43:53.972398+0000","last_active":"2023-10-09T04:48:03.044885+0000","last_peered":"2023-10-09T04:48:03.044885+0000","last_clean":"2023-10-09T04:48:03.044885+0000","last_became_active":"2023-10-09T04:43:53.970226+0000","last_became_peered":"2023-10-09T04:43:53.970226+0000","last_unstale":"2023-10-09T04:48:03.044885+0000","last_undegraded":"2023-10-09T04:48:03.044885+0000","last_fullsized":"2023-10-09T04:48:03.044885+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":154,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.046097+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:48:03.046097+0000","last_peered":"2023-10-09T04:48:03.046097+0000","last_clean":"2023-10-09T04:48:03.046097+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:48:03.046097+0000","last_undegraded":"2023-10-09T04:48:03.046097+0000","last_fullsized":"2023-10-09T04:48:03.046097+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":215,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:04.010678+0000","last_change":"2023-10-09T04:44:51.889935+0000","last_active":"2023-10-09T04:48:04.010678+0000","last_peered":"2023-10-09T04:48:04.010678+0000","last_clean":"2023-10-09T04:48:04.010678+0000","last_became_active":"2023-10-09T04:44:51.889421+0000","last_became_peered":"2023-10-09T04:44:51.889421+0000","last_unstale":"2023-10-09T04:48:04.010678+0000","last_undegraded":"2023-10-09T04:48:04.010678+0000","last_fullsized":"2023-10-09T04:48:04.010678+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:53.165469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":246,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.039330+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:48:03.039330+0000","last_peered":"2023-10-09T04:48:03.039330+0000","last_clean":"2023-10-09T04:48:03.039330+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:48:03.039330+0000","last_undegraded":"2023-10-09T04:48:03.039330+0000","last_fullsized":"2023-10-09T04:48:03.039330+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T04:48:03.044538+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:48:03.044538+0000","last_peered":"2023-10-09T04:48:03.044538+0000","last_clean":"2023-10-09T04:48:03.044538+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:48:03.044538+0000","last_undegraded":"2023-10-09T04:48:03.044538+0000","last_fullsized":"2023-10-09T04:48:03.044538+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2775711744,"num_objects":42354,"num_object_clones":0,"num_object_copies":127062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":42354,"num_write_kb":2710656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8367833088,"data_stored":8367833088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42354,"ondisk_log_size":42354,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430693,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2127088,"kb_used_data":1529192,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91618064,"statfs":{"total":95995035648,"available":93816897536,"internally_reserved":0,"allocated":1565892608,"data_stored":1565309904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":2.4060000000000001,"15min":2.3380000000000001},"min":{"1min":0.44700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":1.0309999999999999,"5min":49.149999999999999,"15min":104.16200000000001},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.85399999999999998,"5min":2.3500000000000001,"15min":2.4399999999999999},"min":{"1min":0.53800000000000003,"5min":0.317,"15min":0.27300000000000002},"max":{"1min":1.331,"5min":48.920999999999999,"15min":104.129},"last":0.747}]},{"osd":1,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":2.5499999999999998,"15min":1.7210000000000001},"min":{"1min":0.61199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.32,"5min":58.816000000000003,"15min":58.816000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.5310000000000001,"15min":1.774},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.7430000000000003,"5min":58.840000000000003,"15min":58.840000000000003},"last":0.89400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.601,"15min":1.748},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0620000000000001,"5min":6.5499999999999998,"15min":6.5499999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.1499999999999999,"15min":2.536},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":3.5270000000000001,"5min":11.583,"15min":11.583},"last":0.86399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":7.0510000000000002,"5min":2.8759999999999999,"15min":2.101},"min":{"1min":0.59499999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":102.02800000000001,"5min":102.02800000000001,"15min":102.02800000000001},"last":102.02800000000001},{"interface":"front","average":{"1min":7.133,"5min":2.887,"15min":1.948},"min":{"1min":0.61199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":101.973,"5min":101.973,"15min":101.973},"last":101.973}]},{"osd":4,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.5109999999999999,"15min":1.278},"min":{"1min":0.47899999999999998,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":3.9510000000000001,"5min":11.933999999999999,"15min":11.933999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.496,"15min":1.3320000000000001},"min":{"1min":0.433,"5min":0.308,"15min":0.308},"max":{"1min":2.4039999999999999,"5min":8.452,"15min":10.696999999999999},"last":0.78100000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.706,"15min":1.514},"min":{"1min":0.43099999999999999,"5min":0.41899999999999998,"15min":0.35699999999999998},"max":{"1min":1.4379999999999999,"5min":11.268000000000001,"15min":11.268000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.9079999999999999,"15min":1.659},"min":{"1min":0.54100000000000004,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.7679999999999998,"5min":15.138999999999999,"15min":15.138999999999999},"last":0.90700000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:48:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.5600000000000001,"15min":1.466},"min":{"1min":0.71399999999999997,"5min":0.42299999999999999,"15min":0.36899999999999999},"max":{"1min":1.4870000000000001,"5min":11.986000000000001,"15min":11.986000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.05,"5min":1.635,"15min":1.337},"min":{"1min":0.56000000000000005,"5min":0.36599999999999999,"15min":0.35599999999999998},"max":{"1min":1.9239999999999999,"5min":12.013999999999999,"15min":12.013999999999999},"last":0.879}]}]},{"osd":6,"up_from":146,"seq":627065225393,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807336,"kb_used_data":477280,"kb_used_omap":5,"kb_used_meta":330042,"kb_avail":92937816,"statfs":{"total":95995035648,"available":95168323584,"internally_reserved":0,"allocated":488734720,"data_stored":488160081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":337963616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.6549999999999998,"15min":3.5150000000000001},"min":{"1min":0.46300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.786,"5min":76.189999999999998,"15min":214.82300000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.72399999999999998,"5min":2.823,"15min":3.5209999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":1.0169999999999999,"5min":76.165000000000006,"15min":213.34700000000001},"last":0.71699999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":4.1459999999999999,"5min":2.7200000000000002,"15min":2.6909999999999998},"min":{"1min":0.39500000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":46.951000000000001,"5min":46.951000000000001,"15min":76.081000000000003},"last":46.951000000000001},{"interface":"front","average":{"1min":3.7429999999999999,"5min":2.113,"15min":2.6110000000000002},"min":{"1min":0.41599999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":45.884999999999998,"5min":45.884999999999998,"15min":75.546000000000006},"last":45.884999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:47:58 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.9119999999999999,"15min":2.0339999999999998},"min":{"1min":0.41699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.0570000000000004,"5min":6.0570000000000004,"15min":6.0570000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.5189999999999999,"15min":1.554},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":3.7360000000000002,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.80800000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.722,"15min":1.726},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.9140000000000001,"5min":6.7599999999999998,"15min":11.673},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.544,"15min":1.724},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.522,"5min":9.3190000000000008,"15min":15.824},"last":0.85699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.673,"15min":1.3009999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.248},"max":{"1min":1.696,"5min":13.699999999999999,"15min":13.699999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.647,"15min":1.431},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.9820000000000002,"5min":7.0570000000000004,"15min":10.324},"last":0.82999999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:48:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.726,"15min":1.5940000000000001},"min":{"1min":0.57999999999999996,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.9940000000000002,"5min":9.3000000000000007,"15min":10.193},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.778,"15min":1.706},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.7989999999999999,"5min":9.9740000000000002,"15min":11.727},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.881,"15min":1.6040000000000001},"min":{"1min":0.42799999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.7650000000000001,"5min":16.527999999999999,"15min":16.527999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.905,"15min":1.526},"min":{"1min":0.61199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.742,"5min":15.366,"15min":15.366},"last":0.88800000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958842,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089468,"kb_used_data":1468596,"kb_used_omap":6,"kb_used_meta":620857,"kb_avail":91655684,"statfs":{"total":95995035648,"available":93855420416,"internally_reserved":0,"allocated":1503842304,"data_stored":1502841931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":635757580},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.4300000000000002,"15min":2.746},"min":{"1min":0.307,"5min":0.19700000000000001,"15min":0.17499999999999999},"max":{"1min":1.0900000000000001,"5min":41.305,"15min":91.025999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.629,"5min":2.4079999999999999,"15min":2.8580000000000001},"min":{"1min":0.316,"5min":0.23899999999999999,"15min":0.17999999999999999},"max":{"1min":1.526,"5min":43.777999999999999,"15min":91.536000000000001},"last":0.98799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:47:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":2.4990000000000001,"15min":2.637},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.153,"5min":15.336,"15min":15.336},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.1440000000000001,"15min":2.4119999999999999},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":5.6580000000000004,"5min":13.444000000000001,"15min":13.444000000000001},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.6299999999999999,"15min":1.8140000000000001},"min":{"1min":0.32900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.0940000000000003,"5min":10.401999999999999,"15min":14.369},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.544,"5min":1.778,"15min":1.8620000000000001},"min":{"1min":0.29699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.9410000000000007,"5min":19.617999999999999,"15min":19.617999999999999},"last":0.95799999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.617,"15min":1.407},"min":{"1min":0.55000000000000004,"5min":0.30499999999999999,"15min":0.29199999999999998},"max":{"1min":1.466,"5min":13.362,"15min":13.362},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.679,"15min":1.4670000000000001},"min":{"1min":0.64000000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.6559999999999999,"5min":8.6890000000000001,"15min":12.209},"last":0.77000000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.9950000000000001,"15min":1.7290000000000001},"min":{"1min":0.55600000000000005,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":5.6920000000000002,"5min":14.494999999999999,"15min":14.494999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.84099999999999997,"5min":2.008,"15min":1.663},"min":{"1min":0.371,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":1.5509999999999999,"5min":13.413,"15min":13.413},"last":0.66500000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.8109999999999999,"15min":1.7809999999999999},"min":{"1min":0.53900000000000003,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.9790000000000001,"5min":14.545,"15min":14.545},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.919,"15min":1.861},"min":{"1min":0.64000000000000001,"5min":0.40500000000000003,"15min":0.39900000000000002},"max":{"1min":2.016,"5min":14.577,"15min":14.577},"last":1.0569999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.8460000000000001,"15min":2.0009999999999999},"min":{"1min":0.58699999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.728,"5min":13.474,"15min":144.613},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.002,"5min":1.6919999999999999,"15min":1.97},"min":{"1min":0.68100000000000005,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.7,"5min":8.6500000000000004,"15min":145.53700000000001},"last":0.63900000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122367,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1294992,"kb_used_data":659208,"kb_used_omap":9,"kb_used_meta":635766,"kb_avail":92450160,"statfs":{"total":95995035648,"available":94668963840,"internally_reserved":0,"allocated":675028992,"data_stored":674014080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":651024881},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:47:47 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.4690000000000001,"15min":2.2400000000000002},"min":{"1min":0.29799999999999999,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":4.2729999999999997,"5min":7.5270000000000001,"15min":223.98500000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.8500000000000001,"15min":2.3679999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19900000000000001},"max":{"1min":5.3840000000000003,"5min":10.500999999999999,"15min":223.13},"last":2.1600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.4020000000000001,"15min":2.4470000000000001},"min":{"1min":0.36499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.7850000000000001,"5min":17.109000000000002,"15min":17.109000000000002},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.294,"5min":2.629,"15min":2.7949999999999999},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.202,"5min":12.57,"15min":12.57},"last":0.76200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:47:47 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.5640000000000001,"15min":1.8560000000000001},"min":{"1min":0.33800000000000002,"5min":0.26700000000000002,"15min":0.23400000000000001},"max":{"1min":5.8330000000000002,"5min":10.933999999999999,"15min":64.622},"last":0.749},{"interface":"front","average":{"1min":1.974,"5min":1.976,"15min":2.0840000000000001},"min":{"1min":0.309,"5min":0.23999999999999999,"15min":0.218},"max":{"1min":7.2779999999999996,"5min":10.398999999999999,"15min":64.540999999999997},"last":0.82899999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.9099999999999999,"15min":1.734},"min":{"1min":0.52400000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.649,"5min":16.84,"15min":61.549999999999997},"last":2.5139999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.7450000000000001,"15min":1.569},"min":{"1min":0.67100000000000004,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":1.833,"5min":39.957999999999998,"15min":61.902999999999999},"last":3.5419999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.762,"15min":2.1280000000000001},"min":{"1min":0.5,"5min":0.35399999999999998,"15min":0.311},"max":{"1min":7.1500000000000004,"5min":7.1500000000000004,"15min":61.972999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.5109999999999999,"15min":2.1110000000000002},"min":{"1min":0.621,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.5720000000000001,"5min":10.215999999999999,"15min":62.417000000000002},"last":0.72399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.4470000000000001,"15min":1.9419999999999999},"min":{"1min":0.41699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.915,"5min":7.5759999999999996,"15min":62.462000000000003},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.107,"5min":1.8500000000000001,"15min":2.2000000000000002},"min":{"1min":0.69299999999999995,"5min":0.39400000000000002,"15min":0.32100000000000001},"max":{"1min":2.6429999999999998,"5min":9.1969999999999992,"15min":61.808},"last":0.89800000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:48:14 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.9710000000000001,"15min":1.9179999999999999},"min":{"1min":0.73499999999999999,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":1.7509999999999999,"5min":7.2519999999999998,"15min":62.006},"last":2.552},{"interface":"front","average":{"1min":1.02,"5min":1.931,"15min":1.8819999999999999},"min":{"1min":0.63100000000000001,"5min":0.372,"15min":0.32300000000000001},"max":{"1min":2.0219999999999998,"5min":10.804,"15min":61.606999999999999},"last":2.4769999999999999}]}]},{"osd":2,"up_from":234,"seq":1005022347313,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":723588,"kb_used_data":510332,"kb_used_omap":13,"kb_used_meta":213234,"kb_avail":93021564,"statfs":{"total":95995035648,"available":95254081536,"internally_reserved":0,"allocated":522579968,"data_stored":521572842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":218352511},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.5609999999999999,"15min":2.9569999999999999},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.754,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":2.4060000000000001,"15min":2.8650000000000002},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.249},"max":{"1min":0.84199999999999997,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.57099999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.427,"15min":2.5670000000000002},"min":{"1min":0.31900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7619999999999996,"5min":7.907,"15min":7.907},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.198,"5min":1.7949999999999999,"15min":1.877},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8780000000000001,"5min":7.8780000000000001,"15min":7.8780000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.6859999999999999,"15min":1.673},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.0720000000000001,"5min":14.340999999999999,"15min":14.340999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.2649999999999999,"5min":2.4870000000000001,"15min":2.7450000000000001},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.4550000000000001,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":0.85699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":2.0339999999999998,"15min":2.137},"min":{"1min":0.47799999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6779999999999999,"5min":6.6520000000000001,"15min":6.6520000000000001},"last":0.88},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.994,"15min":2.0459999999999998},"min":{"1min":0.40999999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":3.3769999999999998,"5min":7.9390000000000001,"15min":7.9390000000000001},"last":0.82999999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.9630000000000001,"15min":2.1659999999999999},"min":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.4910000000000001,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.546,"5min":2.6509999999999998,"15min":2.9039999999999999},"min":{"1min":0.68400000000000005,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":3.706,"5min":12.295999999999999,"15min":12.295999999999999},"last":1.1020000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.1389999999999998,"15min":2.2749999999999999},"min":{"1min":0.54700000000000004,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":3.125,"5min":8.2409999999999997,"15min":8.2409999999999997},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.206,"5min":1.724,"15min":1.7070000000000001},"min":{"1min":0.436,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.6240000000000001,"5min":7.3040000000000003,"15min":7.3040000000000003},"last":0.747}]},{"osd":7,"last update":"Mon Oct 9 04:47:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":2.3359999999999999,"15min":2.3039999999999998},"min":{"1min":0.56699999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":14.161,"5min":14.161,"15min":14.161},"last":1.014},{"interface":"front","average":{"1min":1.365,"5min":2.0859999999999999,"15min":2.0539999999999998},"min":{"1min":0.56799999999999995,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":7.2709999999999999,"5min":10.996,"15min":10.996},"last":0.81899999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631788,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2021320,"kb_used_data":1362048,"kb_used_omap":5,"kb_used_meta":659258,"kb_avail":91723832,"statfs":{"total":95995035648,"available":93925203968,"internally_reserved":0,"allocated":1394737152,"data_stored":1393726986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":675080366},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.4770000000000001,"15min":2.8010000000000002},"min":{"1min":0.24299999999999999,"5min":0.19500000000000001,"15min":0.154},"max":{"1min":1.252,"5min":8.5220000000000002,"15min":341.28699999999998},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":1.474,"15min":2.7450000000000001},"min":{"1min":0.28999999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.3169999999999999,"5min":9.5079999999999991,"15min":341.57799999999997},"last":0.56100000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.464,"15min":1.653},"min":{"1min":0.26500000000000001,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":4.1609999999999996,"5min":17.172999999999998,"15min":47.637999999999998},"last":2.3119999999999998},{"interface":"front","average":{"1min":1.159,"5min":2.0099999999999998,"15min":1.8759999999999999},"min":{"1min":0.23200000000000001,"5min":0.21299999999999999,"15min":0.188},"max":{"1min":5.8529999999999998,"5min":17.125,"15min":47.616999999999997},"last":1.2749999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:47:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.591,"15min":1.663},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7409999999999997,"5min":6.4240000000000004,"15min":6.4240000000000004},"last":2.1539999999999999},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.8220000000000001,"15min":1.8959999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3150000000000004,"5min":10.154999999999999,"15min":10.154999999999999},"last":0.53400000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.506,"15min":1.375},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":1.2809999999999999,"5min":8.4350000000000005,"15min":12.981},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.716,"15min":1.452},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.28000000000000003},"max":{"1min":1.335,"5min":9.4359999999999999,"15min":9.4359999999999999},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":2.0329999999999999,"15min":1.728},"min":{"1min":0.53400000000000003,"5min":0.35399999999999998,"15min":0.32300000000000001},"max":{"1min":7.2859999999999996,"5min":7.9649999999999999,"15min":8.0109999999999992},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.427,"5min":1.746,"15min":1.742},"min":{"1min":0.626,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":7.21,"5min":9.4800000000000004,"15min":9.4800000000000004},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.2669999999999999,"15min":1.669},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":1.5189999999999999,"5min":9.0020000000000007,"15min":11.842000000000001},"last":2.2050000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.4630000000000001,"15min":1.5649999999999999},"min":{"1min":0.497,"5min":0.38700000000000001,"15min":0.35399999999999998},"max":{"1min":1.478,"5min":8.4860000000000007,"15min":13.004},"last":0.497}]},{"osd":7,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.6639999999999999,"15min":2.524},"min":{"1min":0.65500000000000003,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":3.718,"5min":6.7729999999999997,"15min":198.012},"last":2.2370000000000001},{"interface":"front","average":{"1min":1.325,"5min":1.6970000000000001,"15min":2.4529999999999998},"min":{"1min":0.46700000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.0449999999999999,"5min":8.4469999999999992,"15min":198.05199999999999},"last":0.46700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019390,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1104296,"kb_used_data":506400,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92640856,"statfs":{"total":95995035648,"available":94864236544,"internally_reserved":0,"allocated":518553600,"data_stored":517978474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":2.52,"15min":1.738},"min":{"1min":0.49299999999999999,"5min":0.26500000000000001,"15min":0.26100000000000001},"max":{"1min":6.6600000000000001,"5min":52.106999999999999,"15min":52.106999999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.86599999999999999,"5min":2.5449999999999999,"15min":1.9930000000000001},"min":{"1min":0.49199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":1.262,"5min":52.289999999999999,"15min":52.289999999999999},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:48:34 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.879,"15min":1.498},"min":{"1min":0.65900000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.4119999999999999,"5min":24.329999999999998,"15min":24.329999999999998},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.032,"5min":2.254,"15min":1.5800000000000001},"min":{"1min":0.51700000000000002,"5min":0.315,"15min":0.315},"max":{"1min":2.738,"5min":24.741,"15min":24.741},"last":1.3260000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:47:59 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.9059999999999999,"15min":2.0059999999999998},"min":{"1min":0.48999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.249000000000001,"5min":10.249000000000001,"15min":10.249000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.556,"15min":2.4630000000000001},"min":{"1min":0.55200000000000005,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.535,"5min":10.535,"15min":10.535},"last":0.60899999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:48:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":1.542,"15min":1.456},"min":{"1min":0.47899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.1180000000000001,"5min":10.42,"15min":10.42},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.05,"5min":1.5960000000000001,"15min":1.4179999999999999},"min":{"1min":0.41299999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.7869999999999999,"5min":8.4909999999999997,"15min":9.484},"last":0.66900000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:48:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":2.1829999999999998,"15min":1.385},"min":{"1min":0.58599999999999997,"5min":0.251,"15min":0.251},"max":{"1min":6.7249999999999996,"5min":9.0549999999999997,"15min":10.412000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.7969999999999999,"15min":1.401},"min":{"1min":0.39200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.4179999999999999,"5min":8.9209999999999994,"15min":8.9209999999999994},"last":0.78000000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:48:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.472,"15min":1.2509999999999999},"min":{"1min":0.48099999999999998,"5min":0.375,"15min":0.375},"max":{"1min":1.1919999999999999,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.472,"15min":1.2669999999999999},"min":{"1min":0.51000000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.46,"5min":8.5470000000000006,"15min":10.257999999999999},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 04:48:34 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.556,"15min":1.2629999999999999},"min":{"1min":0.45300000000000001,"5min":0.34699999999999998,"15min":0.317},"max":{"1min":6.5220000000000002,"5min":7.4029999999999996,"15min":9.0370000000000008},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.6180000000000001,"15min":1.3919999999999999},"min":{"1min":0.58599999999999997,"5min":0.377,"15min":0.35099999999999998},"max":{"1min":2.7189999999999999,"5min":8.9670000000000005,"15min":13.349},"last":0.71099999999999997}]}]},{"osd":5,"up_from":150,"seq":644245094572,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1894040,"kb_used_data":1540240,"kb_used_omap":6,"kb_used_meta":353785,"kb_avail":91851112,"statfs":{"total":95995035648,"available":94055538688,"internally_reserved":0,"allocated":1577205760,"data_stored":1576642466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":362276115},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.9870000000000001,"15min":1.698},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8809999999999998,"5min":19.001000000000001,"15min":19.001000000000001},"last":0.505},{"interface":"front","average":{"1min":0.94099999999999995,"5min":2.1760000000000002,"15min":1.8859999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30099999999999999},"max":{"1min":2.6379999999999999,"5min":18.119,"15min":18.119},"last":0.82899999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.7110000000000001,"15min":1.6739999999999999},"min":{"1min":0.55000000000000004,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.3370000000000002,"5min":9.4309999999999992,"15min":9.4309999999999992},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.04,"5min":1.879,"15min":1.7889999999999999},"min":{"1min":0.496,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9529999999999998,"5min":11.311999999999999,"15min":11.311999999999999},"last":1.335}]},{"osd":2,"last update":"Mon Oct 9 04:47:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.714,"15min":1.7050000000000001},"min":{"1min":0.47699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.7670000000000003,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.585,"5min":2.0870000000000002,"15min":2.327},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.9139999999999997,"5min":13.536,"15min":13.536},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.5820000000000001,"15min":1.573},"min":{"1min":0.56499999999999995,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.1190000000000002,"5min":9.3889999999999993,"15min":9.3889999999999993},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.8999999999999999,"15min":1.9419999999999999},"min":{"1min":0.47399999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.7559999999999998,"5min":11.273,"15min":11.273},"last":1.524}]},{"osd":4,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.7929999999999999,"15min":1.542},"min":{"1min":0.44,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.7569999999999999,"5min":11.359,"15min":11.359},"last":1.552},{"interface":"front","average":{"1min":0.89200000000000002,"5min":1.704,"15min":1.3600000000000001},"min":{"1min":0.32300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.349,"5min":10.709,"15min":10.709},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.784,"15min":1.52},"min":{"1min":0.38600000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.375,"5min":10.77,"15min":11.087},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.081,"5min":1.8049999999999999,"15min":1.5780000000000001},"min":{"1min":0.66900000000000004,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.298,"5min":11.222,"15min":11.222},"last":3.7999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.8280000000000001,"15min":1.804},"min":{"1min":0.73499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.9690000000000001,"5min":11.185,"15min":39.771000000000001},"last":4.2350000000000003},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.02,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3140000000000001,"5min":14.170999999999999,"15min":39.905000000000001},"last":0.74099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":685834240,"data_stored":685834240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1554644992,"data_stored":1554644992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":531628032,"data_stored":531628032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1396506624,"data_stored":1396506624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":517472256,"data_stored":517472256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1598816256,"data_stored":1598816256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":491978752,"data_stored":491978752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1590951936,"data_stored":1590951936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:48:56.498 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:48:56.499 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:48:56.499 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T04:48:56.499 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:48:56.499 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:48:56.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T04:48:56.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:56.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:56.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:56.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:56.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:56.854 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:56.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:56.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:56.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:57.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:57.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:57.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 51678 51662 107.615 75.9375 0.0097363 0.00928717 2023-10-09T04:48:57.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:57.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:57.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:57.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:57.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:57.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:57.634 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:57.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:57.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:57.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:57.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:57.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:57.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:58.032 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:58.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:58.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:58.169 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:48:58 smithi082 bash[77063]: cluster 2023-10-09T04:48:56.717748+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:48:58.170 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:48:58 smithi082 bash[79733]: cluster 2023-10-09T04:48:56.717748+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:48:58.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:58.204 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 52997 52981 106.803 82.4375 0.0248983 0.00935764 2023-10-09T04:48:58.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:58.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:48:58 smithi153 bash[59565]: cluster 2023-10-09T04:48:56.717748+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:48:58.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:58.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:58.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:48:58.490 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:48:58.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:58.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:58.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:58.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:58.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:58.845 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:58.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:58.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:58.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:58.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:59.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:59.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:59.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:59.205 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 54236 54220 105.885 77.4375 0.020536 0.0094385 2023-10-09T04:48:59.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:59.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:59.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.538 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:59.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:59.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:59.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:59.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:59.783 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:48:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:48:59] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T04:48:59.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:59.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:59.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:48:59.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:00.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:00.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:00 smithi153 bash[59565]: cluster 2023-10-09T04:48:58.718294+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:49:00.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:00 smithi153 bash[59565]: audit 2023-10-09T04:48:59.290232+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:00.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:00.205 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 55518 55502 105.101 80.125 0.0102515 0.00950867 2023-10-09T04:49:00.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:00.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:00.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:00.463 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:00 smithi082 bash[77063]: cluster 2023-10-09T04:48:58.718294+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:49:00.463 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:00 smithi082 bash[77063]: audit 2023-10-09T04:48:59.290232+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:00.463 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:00 smithi082 bash[79733]: cluster 2023-10-09T04:48:58.718294+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:49:00.464 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:00 smithi082 bash[79733]: audit 2023-10-09T04:48:59.290232+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:00.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:00.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.632 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:00.736 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:00.771 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:00.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:00.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:01.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:01.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:01.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:01.205 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 56780 56764 104.33 78.875 0.00623031 0.00958006 2023-10-09T04:49:01.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:01.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:01.372 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:01.372 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:01.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:01.450 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:01.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:01.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:01.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:01.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:01.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:01.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:01.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:02.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:02.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:02.111 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:02 smithi082 bash[77063]: cluster 2023-10-09T04:49:00.719126+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:49:02.112 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:02 smithi082 bash[79733]: cluster 2023-10-09T04:49:00.719126+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:49:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.140 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:02.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:02.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 58029 58013 103.579 78.0625 0.00922999 0.00964907 2023-10-09T04:49:02.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:02.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:02.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:02.392 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:02.395 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:02.400 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:02 smithi153 bash[59565]: cluster 2023-10-09T04:49:00.719126+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:49:02.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:02.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:02.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:02.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:02.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:02.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:02.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:02.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:02.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:03.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 59253 59237 102.827 76.5 0.0101285 0.00971898 2023-10-09T04:49:03.207 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:03.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.343 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.447 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:03.601 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:03.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.863 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:03.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:03.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:04.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:04.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:04.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:04.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 60377 60361 101.946 70.25 0.0930785 0.00978779 2023-10-09T04:49:04.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:04.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:04.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.380 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:04 smithi082 bash[77063]: cluster 2023-10-09T04:49:02.719629+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:49:04.380 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:04 smithi082 bash[79733]: cluster 2023-10-09T04:49:02.719629+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:49:04.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:04 smithi153 bash[59565]: cluster 2023-10-09T04:49:02.719629+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1120: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:49:04.502 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.532 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:04.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:04.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:04.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:04.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:04.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:05.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:05.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:05.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:05.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:05.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 61588 61572 101.255 75.6875 0.00995595 0.00986999 2023-10-09T04:49:05.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:05.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:05.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:05.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:05.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:05.591 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:05.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:05.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:05.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:05.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:05.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:05.961 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:05.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:05.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:06.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:06.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:06.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 62797 62781 100.596 75.5625 0.0049955 0.00993415 2023-10-09T04:49:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:06 smithi082 bash[77063]: cluster 2023-10-09T04:49:04.720253+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:49:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:06 smithi082 bash[79733]: cluster 2023-10-09T04:49:04.720253+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:49:06.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:06.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:06.376 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:06.385 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:06 smithi153 bash[59565]: cluster 2023-10-09T04:49:04.720253+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:49:06.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:06.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:06.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:06.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:06.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:06.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:06.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:06.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:06.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:06.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:06.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:06.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:07.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:07.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:07.193 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:07.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:49:07.205368+0000 min lat: 0.00135091 max lat: 0.247697 avg lat: 0.00999979 2023-10-09T04:49:07.305 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:49:07.305 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 63992 63976 99.9482 74.6875 0.00462178 0.00999979 2023-10-09T04:49:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.306 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:07.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:07.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:07.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:07.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:07.561 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:07.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:07.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:07.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:07.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:07.931 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:07.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:07.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:08.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.089 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:08.136 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:08 smithi082 bash[79733]: cluster 2023-10-09T04:49:06.721634+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:49:08.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:08 smithi082 bash[77063]: cluster 2023-10-09T04:49:06.721634+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:49:08.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:08.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 65048 65032 99.12 66 0.0169619 0.0100829 2023-10-09T04:49:08.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:08.324 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:08.335 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:08 smithi153 bash[59565]: cluster 2023-10-09T04:49:06.721634+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:49:08.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:08.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:08.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:08.632 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:08.738 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:08.766 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:08.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:08.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:08.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:09.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:09.132 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:09.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:09.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 66190 66174 98.4592 71.375 0.00551795 0.0101494 2023-10-09T04:49:09.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:09.298 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:09.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:09.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:09.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:09.569 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:09.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:09.700 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:09] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T04:49:09.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:09.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:09.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:09.941 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:09.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:09.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:09.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:10.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:10.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:10.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:10.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 67231 67215 97.6825 65.0625 0.0101586 0.010232 2023-10-09T04:49:10.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:10.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:10 smithi082 bash[77063]: cluster 2023-10-09T04:49:08.722474+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:10.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:10 smithi082 bash[79733]: cluster 2023-10-09T04:49:08.722474+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:10.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:10.340 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:10.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:10 smithi153 bash[59565]: cluster 2023-10-09T04:49:08.722474+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:10.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:10.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:10.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:10.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:10.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:10.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:10.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:10.913 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:10.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:11.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:11.053 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:11.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:11.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:11.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 68389 68373 97.1072 72.375 0.0148146 0.0102927 2023-10-09T04:49:11.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:11.319 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:11.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:11.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:11.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:11.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:11.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:11.719 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:11.822 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:11.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:11.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:11.982 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:11.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:12.085 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:12.098 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:12 smithi153 bash[59565]: cluster 2023-10-09T04:49:10.723219+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:49:12.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:12.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 69204 69188 96.0811 50.9375 0.0129672 0.0104025 2023-10-09T04:49:12.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:12.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:12.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:12.355 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:12.387 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:12.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:12 smithi082 bash[77063]: cluster 2023-10-09T04:49:10.723219+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:49:12.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:12 smithi082 bash[79733]: cluster 2023-10-09T04:49:10.723219+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:49:12.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:12.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:12.621 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:12.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:12.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:12.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:12.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:12.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:12.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:13.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:13.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:13.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:13.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:13.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:13.206 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 70271 70255 95.442 66.6875 0.016277 0.0104719 2023-10-09T04:49:13.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:13.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:13.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:13.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:13.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:13.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:13.627 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:13.731 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:13.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:13.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:13.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:13.892 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:13.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:13.996 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:14.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:14.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:14.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:14.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 16 71172 71156 94.6094 56.3125 0.0134985 0.0105641 2023-10-09T04:49:14.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:14.266 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:14 smithi153 bash[59565]: cluster 2023-10-09T04:49:12.723687+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:14.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:14.333 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:14 smithi082 bash[77063]: cluster 2023-10-09T04:49:12.723687+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:14.334 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:14 smithi082 bash[79733]: cluster 2023-10-09T04:49:12.723687+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:49:14.390 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:14.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:14.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:14.580 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:14.684 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:14.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:14.822 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:14.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:14.914 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:49:14.914 INFO:teuthology.orchestra.run.smithi082.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:49:14.914 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:49:14.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:14.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:14.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:15.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:15.121 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:15.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:15.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 72166 72150 93.9326 62.125 0.00941363 0.0106412 2023-10-09T04:49:15.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:15.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:15 smithi082 bash[77063]: audit 2023-10-09T04:49:14.284060+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:15.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:15 smithi082 bash[77063]: audit 2023-10-09T04:49:14.290460+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:15.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:15 smithi082 bash[79733]: audit 2023-10-09T04:49:14.284060+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:15.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:15 smithi082 bash[79733]: audit 2023-10-09T04:49:14.290460+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:15.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:15.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:15.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:15.367 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:15 smithi153 bash[59565]: audit 2023-10-09T04:49:14.284060+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:15.367 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:15 smithi153 bash[59565]: audit 2023-10-09T04:49:14.290460+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:15.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:15.503 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:15.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:15.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:15.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:15.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:15.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:15.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:15.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:15.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:16.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:16.053 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:16 smithi153 bash[59565]: cluster 2023-10-09T04:49:14.724519+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:49:16.058 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:16.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:16.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:16.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 73125 73109 93.2387 59.9375 0.00499217 0.0107184 2023-10-09T04:49:16.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:16.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:16 smithi082 bash[77063]: cluster 2023-10-09T04:49:14.724519+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:49:16.313 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:16 smithi082 bash[79733]: cluster 2023-10-09T04:49:14.724519+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:49:16.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:16.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:16.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:16.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:16.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:16.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:16.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:16.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:16.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:16.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:16.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:17.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:17.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:17.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:17.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 74220 74204 92.7426 68.4375 0.0131694 0.0107772 2023-10-09T04:49:17.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:17.307 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:17.413 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:17.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:17.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:17.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:17.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:17.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:17.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:17.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:17.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:17.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:17.990 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:17.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:18.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:18.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:18.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 75238 75222 92.1715 63.625 0.019281 0.0108434 2023-10-09T04:49:18.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:18.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:18 smithi082 bash[77063]: cluster 2023-10-09T04:49:16.725181+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:49:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:18 smithi082 bash[79733]: cluster 2023-10-09T04:49:16.725181+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:49:18.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:18.287 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:18.287 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:18.354 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:18.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:18.407 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:18 smithi153 bash[59565]: cluster 2023-10-09T04:49:16.725181+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:49:18.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:18.511 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:18.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:18.537 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:49:18.602 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:49:18.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:18.650 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:49:18.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:18.699 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:49:18.739 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:49:18.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:18.778 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:49:18.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:18.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:18.815 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:49:18.860 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:49:18.866 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.866 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.866 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.866 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.867 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.867 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.867 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.867 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:49:18.902 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:18.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:18.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:19.032 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.062 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:19.072 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:49:19.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:19 smithi153 bash[59565]: audit 2023-10-09T04:49:18.487304+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.82:0/3112427634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:19.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:19 smithi153 bash[59565]: audit 2023-10-09T04:49:18.878078+0000 mon.a (mon.0) 1536 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:19.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:19.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 76154 76138 91.4999 57.25 0.00815344 0.0109233 2023-10-09T04:49:19.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:19.331 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:19.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:19 smithi082 bash[77063]: audit 2023-10-09T04:49:18.487304+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.82:0/3112427634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:19.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:19 smithi082 bash[77063]: audit 2023-10-09T04:49:18.878078+0000 mon.a (mon.0) 1536 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:19.382 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:19 smithi082 bash[79733]: audit 2023-10-09T04:49:18.487304+0000 mon.c (mon.2) 116 : audit 0 from='client.? 172.21.15.82:0/3112427634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:19.382 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:19 smithi082 bash[79733]: audit 2023-10-09T04:49:18.878078+0000 mon.a (mon.0) 1536 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:19.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:19.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:19.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:19.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:19.756 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:19.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:19.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:19.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:19.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:20.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:49:20.033 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:19] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T04:49:20.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:20.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:18.725847+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T04:49:20.074 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.055867+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.062762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.063049+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: audit 2023-10-09T04:49:19.064433+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.064690+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.064776+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.067438+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.068269+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.069773+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.075 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:20 smithi153 bash[59565]: cluster 2023-10-09T04:49:19.070562+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:20.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:20.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 76958 76942 90.7216 50.25 0.0133574 0.0110179 2023-10-09T04:49:20.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:20.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:18.725847+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T04:49:20.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.055867+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:20.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.062762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.063049+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: audit 2023-10-09T04:49:19.064433+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.064690+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.064776+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.067438+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.068269+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.069773+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[79733]: cluster 2023-10-09T04:49:19.070562+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:18.725847+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.055867+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.062762+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.063049+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: audit 2023-10-09T04:49:19.064433+0000 mon.a (mon.0) 1538 : audit 1 from='client.? 172.21.15.82:0/2041823161' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.064690+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.064776+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.067438+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.068269+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.069773+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:20 smithi082 bash[77063]: cluster 2023-10-09T04:49:19.070562+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T04:49:20.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:20.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:20.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:20.595 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:20.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:20.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:20.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.865 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:20.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:20.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:21.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:21.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:21.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 77905 77889 90.1375 59.1875 0.0216371 0.0110887 2023-10-09T04:49:21.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:21.274 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:21.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:21.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:21.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:21.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:21.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:21.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:21.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:21.811 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:21.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:21.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:21.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:21.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:21.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[77063]: cluster 2023-10-09T04:49:20.726604+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T04:49:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[77063]: audit 2023-10-09T04:49:20.846140+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:21.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[77063]: audit 2023-10-09T04:49:20.856618+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[79733]: cluster 2023-10-09T04:49:20.726604+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T04:49:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[79733]: audit 2023-10-09T04:49:20.846140+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:21.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:21 smithi082 bash[79733]: audit 2023-10-09T04:49:20.856618+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:22.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:22.079 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:21 smithi153 bash[59565]: cluster 2023-10-09T04:49:20.726604+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T04:49:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:21 smithi153 bash[59565]: audit 2023-10-09T04:49:20.846140+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:22.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:21 smithi153 bash[59565]: audit 2023-10-09T04:49:20.856618+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:22.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:22.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 78954 78938 89.6907 65.5625 0.01775 0.0111441 2023-10-09T04:49:22.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:22.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:22.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:22.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:22.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:22.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:22.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:22.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:22.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:22.759 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:49:22.759 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:49:22.759 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T04:49:22.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:49:22.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:22.880 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:22.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:22.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:23.011 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:23.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:23.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:23.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:23.207 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 79961 79945 89.2129 62.9375 0.0156839 0.0112036 2023-10-09T04:49:23.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:23.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:23.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:23.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:23.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:23.584 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:23.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:23.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:23.736 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:23.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:23.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:23.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:23.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:23 smithi082 bash[77063]: cluster 2023-10-09T04:49:22.728233+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:23.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:23 smithi082 bash[79733]: cluster 2023-10-09T04:49:22.728233+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:23.983 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:24.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:24.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:49:24.098 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:49:24.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:24.133 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:23 smithi153 bash[59565]: cluster 2023-10-09T04:49:22.728233+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:24.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:24.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 80882 80866 88.6541 57.5625 0.0239803 0.0112733 2023-10-09T04:49:24.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:24.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:24.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:24.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:24.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:24.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:24.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:24.572 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:24.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:24.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:24.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:24.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:24.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:24.977 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:25.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:25.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:25.136 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:25.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 81742 81726 88.0523 53.75 0.0109852 0.0113513 2023-10-09T04:49:25.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:25.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:25.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:25.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:25.519 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:25.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:25.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:25.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:25.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:25.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:25.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:25.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:25.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:25.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:26.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:26.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:26.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 82682 82666 87.5556 58.75 0.00898196 0.0114157 2023-10-09T04:49:26.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: cluster 2023-10-09T04:49:24.729603+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: audit 2023-10-09T04:49:25.101358+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: audit 2023-10-09T04:49:25.114086+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: audit 2023-10-09T04:49:25.797195+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: audit 2023-10-09T04:49:25.798518+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[77063]: audit 2023-10-09T04:49:25.807434+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: cluster 2023-10-09T04:49:24.729603+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T04:49:26.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: audit 2023-10-09T04:49:25.101358+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: audit 2023-10-09T04:49:25.114086+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: audit 2023-10-09T04:49:25.797195+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:26.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: audit 2023-10-09T04:49:25.798518+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:26.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:26 smithi082 bash[79733]: audit 2023-10-09T04:49:25.807434+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.250 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:26.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: cluster 2023-10-09T04:49:24.729603+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T04:49:26.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: audit 2023-10-09T04:49:25.101358+0000 mon.a (mon.0) 1542 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: audit 2023-10-09T04:49:25.114086+0000 mon.a (mon.0) 1543 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: audit 2023-10-09T04:49:25.797195+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:26.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: audit 2023-10-09T04:49:25.798518+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:26.362 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:26 smithi153 bash[59565]: audit 2023-10-09T04:49:25.807434+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:49:26.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:26.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:26.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:26.656 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:26.657 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:26.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:26.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:26.884 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:26.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:26.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:27.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:27.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:27.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:27.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:49:27.209385+0000 min lat: 0.00135091 max lat: 0.280251 avg lat: 0.0114741 2023-10-09T04:49:27.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:49:27.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 83656 83640 87.1108 60.875 0.00589622 0.0114741 2023-10-09T04:49:27.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:27.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:27.430 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:27.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:27.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:27.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:27.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:27.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:27.833 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:27.859 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:27.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:27.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:28.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:28.106 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:28 smithi153 bash[59565]: cluster 2023-10-09T04:49:26.730436+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T04:49:28.121 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:28.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 84540 84524 86.5884 55.25 0.00873288 0.0115326 2023-10-09T04:49:28.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:28.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:28.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:28.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:28.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:28 smithi082 bash[77063]: cluster 2023-10-09T04:49:26.730436+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T04:49:28.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:28 smithi082 bash[79733]: cluster 2023-10-09T04:49:26.730436+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T04:49:28.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:28.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:28.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:28.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:28.686 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:28.686 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:28.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:28.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:28.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:28.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:28.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:29.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:29.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:29.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:29.201 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:29.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 85494 85478 86.1535 59.625 0.00722742 0.0116015 2023-10-09T04:49:29.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:29.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:29.454 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:29.483 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:29.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:29.621 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:29.725 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:29.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:29.859 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:29.884 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:29.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:29] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T04:49:29.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:30.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:30.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:30.137 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:30 smithi153 bash[59565]: cluster 2023-10-09T04:49:28.730961+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T04:49:30.138 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:30 smithi153 bash[59565]: audit 2023-10-09T04:49:29.290857+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:30.161 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:30.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 86425 86409 85.7095 58.1875 0.00365934 0.0116519 2023-10-09T04:49:30.267 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:30.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:30 smithi082 bash[77063]: cluster 2023-10-09T04:49:28.730961+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T04:49:30.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:30 smithi082 bash[77063]: audit 2023-10-09T04:49:29.290857+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:30 smithi082 bash[79733]: cluster 2023-10-09T04:49:28.730961+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2023-10-09T04:49:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:30 smithi082 bash[79733]: audit 2023-10-09T04:49:29.290857+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:30.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:30.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:30.742 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:30.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:30.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:30.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:30.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:30.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:31.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:31.105 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:31.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 87405 87389 85.3272 61.25 0.0202744 0.0117146 2023-10-09T04:49:31.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:31.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:31.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:31.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:31.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:31.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:31.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:31.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:31.790 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:31.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:31.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:32.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:32.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:32.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:32.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 88301 88285 84.8759 56 0.0181521 0.0117766 2023-10-09T04:49:32.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:32.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:32 smithi082 bash[77063]: cluster 2023-10-09T04:49:30.731610+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:49:32.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:32 smithi082 bash[79733]: cluster 2023-10-09T04:49:30.731610+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:49:32.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:32.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:32.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:32.431 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:32 smithi153 bash[59565]: cluster 2023-10-09T04:49:30.731610+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1135: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:49:32.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:32.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:32.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:32.658 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:32.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:32.792 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:32.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:32.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:32.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:33.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.068 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:33.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:33.210 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 89251 89235 84.4895 59.375 0.0214165 0.01183 2023-10-09T04:49:33.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:33.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:33.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:33.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:33.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:33.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:33.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:33.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:33.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:33.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:33.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:33.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:34.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:34.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:34.211 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 90116 90100 84.0353 54.0625 0.0158111 0.0118939 2023-10-09T04:49:34.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:34.248 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:34 smithi153 bash[59565]: cluster 2023-10-09T04:49:32.732132+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:34.269 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:34.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:34.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:34 smithi082 bash[77063]: cluster 2023-10-09T04:49:32.732132+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:34 smithi082 bash[79733]: cluster 2023-10-09T04:49:32.732132+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:34.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:34.535 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:34.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:34.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:34.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:34.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:34.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:34.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:34.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:35.039 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:35.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:35.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:35.211 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 90907 90891 83.5265 49.4375 0.0208745 0.0119669 2023-10-09T04:49:35.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:35.433 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:35.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:35.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:35.703 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:35.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:35.952 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:35.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:35.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:36.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:36.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:36.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:36.213 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 15 91857 91842 83.174 59.4375 0.0286402 0.0120173 2023-10-09T04:49:36.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:36.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:36.325 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:36 smithi082 bash[77063]: cluster 2023-10-09T04:49:34.732729+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:36.326 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:36 smithi082 bash[79733]: cluster 2023-10-09T04:49:34.732729+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:36.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:36 smithi153 bash[59565]: cluster 2023-10-09T04:49:34.732729+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:49:36.350 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:36.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:36.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:36.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:36.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:36.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:36.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:36.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:36.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:36.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:36.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:36.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:37.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:37.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:37.183 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:37.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 92792 92776 82.8197 58.375 0.0250301 0.0120686 2023-10-09T04:49:37.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:37.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:37.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:37.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:37.484 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:37.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:37.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:37.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:37.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:37.852 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:37.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:37.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:37.990 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:38.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:38.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:38.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:38.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:38.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 16 93712 93696 82.463 57.5 0.0164163 0.0121206 2023-10-09T04:49:38.252 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:38 smithi082 bash[77063]: cluster 2023-10-09T04:49:36.733560+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:49:38.252 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:38 smithi082 bash[79733]: cluster 2023-10-09T04:49:36.733560+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:49:38.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:38.307 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:38.412 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:38.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:38 smithi153 bash[59565]: cluster 2023-10-09T04:49:36.733560+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:49:38.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:38.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:38.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:38.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:38.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:38.718 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:38.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:38.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:38.863 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:38.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:38.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:39.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:39.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:39.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:39.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 16 94617 94601 82.1032 56.5625 0.0129112 0.0121743 2023-10-09T04:49:39.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:39 smithi082 bash[77063]: cluster 2023-10-09T04:49:38.734734+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:39.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:39 smithi082 bash[79733]: cluster 2023-10-09T04:49:38.734734+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:39.243 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:39.271 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:39.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:39.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:39.514 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:39.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:39 smithi153 bash[59565]: cluster 2023-10-09T04:49:38.734734+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:39.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:39.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:39.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:39.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:39.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:39.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:39.921 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:39.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:39.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:40.044 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:39] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T04:49:40.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:40.099 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:40.202 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:40.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 95465 95449 81.7046 53 0.0389205 0.012234 2023-10-09T04:49:40.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:40.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:40.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:40.484 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:40.517 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:40.623 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:40.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:40.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:40.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:40.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:40.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:40.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:40.947 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:49:40.954 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:49:41.020 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:41.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:41.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:41.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:41.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 96300 96284 81.3056 52.1875 0.0123941 0.0122936 2023-10-09T04:49:41.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:41.319 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:41.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:41.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:41.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:41.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:41.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:41.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:41.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:41.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:41.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:41.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:41.997 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:42.103 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:42.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:42.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 97215 97199 80.984 57.1875 0.0124457 0.012342 2023-10-09T04:49:42.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:42 smithi082 bash[77063]: cluster 2023-10-09T04:49:40.735431+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:49:42.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:42 smithi082 bash[77063]: audit 2023-10-09T04:49:40.922335+0000 mgr.y (mgr.34107) 1070 : audit 0 from='client.45650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:42 smithi082 bash[79733]: cluster 2023-10-09T04:49:40.735431+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:49:42.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:42 smithi082 bash[79733]: audit 2023-10-09T04:49:40.922335+0000 mgr.y (mgr.34107) 1070 : audit 0 from='client.45650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:42.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:42.263 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:42 smithi153 bash[59565]: cluster 2023-10-09T04:49:40.735431+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:49:42.263 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:42 smithi153 bash[59565]: audit 2023-10-09T04:49:40.922335+0000 mgr.y (mgr.34107) 1070 : audit 0 from='client.45650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:42.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:42.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:42.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:42.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:42.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:42.797 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:42.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:42.962 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:42.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:43.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:43.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:43.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:43.191 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:43.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 97980 97964 80.5475 47.8125 0.0220869 0.0124071 2023-10-09T04:49:43.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:43.355 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:43.382 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:43.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:43.519 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:43.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:43.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:43.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:43.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:43.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:43.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:43.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:44.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:44.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:44.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:44.191 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:44.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 16 98730 98714 80.1102 46.875 0.0117242 0.0124777 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:148898 8.f 8.4d6cec3f (undecoded) ondisk+retry+write+known_if_redirected e234)", 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:50.894461+0000", 2023-10-09T04:49:44.294 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 293.39700527600002, 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98603037999999998, 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 148898 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894461+0000", 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894461+0000", 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894458+0000", 2023-10-09T04:49:44.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894466+0000", 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7360000000000001e-06 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894475+0000", 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5189999999999998e-06 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.894487+0000", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1970999999999999e-05 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.895201+0000", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00071376800000000004 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.895211+0000", 2023-10-09T04:49:44.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5820000000000008e-06 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880477+0000", 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.985266631 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880491+0000", 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3587000000000001e-05 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.299 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:148934 8.f 8.2ac8c90f (undecoded) ondisk+write+known_if_redirected e234)", 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:50.938185+0000", 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 293.35328055999997, 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94233876299999997, 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:49:44.300 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 148934 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938185+0000", 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938185+0000", 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938183+0000", 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999976 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938195+0000", 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1493e-05 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938213+0000", 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8796000000000001e-05 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938224+0000", 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0753e-05 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938274+0000", 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9916000000000001e-05 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:49:44.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.938281+0000", 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.267e-06 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880516+0000", 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94223457700000002 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880524+0000", 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1839999999999999e-06 2023-10-09T04:49:44.306 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35442.0:148940 8.f 8.b9d3004f (undecoded) ondisk+write+known_if_redirected e234)", 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:50.952650+0000", 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 293.33881623000002, 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.92790206799999997, 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:49:44.307 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35442", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 148940 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952650+0000", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952650+0000", 2023-10-09T04:49:44.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952648+0000", 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952656+0000", 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3969999999999994e-06 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952671+0000", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4741e-05 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952679+0000", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.038e-06 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952725+0000", 2023-10-09T04:49:44.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.5804999999999999e-05 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:50.952732+0000", 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3989999999999998e-06 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880544+0000", 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.92781199199999997 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:51.880552+0000", 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.3830000000000003e-06 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:151888 8.e e236/235 8:7bb8d1bf:::benchmark_data_smithi082_175677_object34874:head v 236'9568, mlcod=223'2174)", 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.287470+0000", 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 286.00399602700003, 2023-10-09T04:49:44.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.14030755, 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287470+0000", 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287470+0000", 2023-10-09T04:49:44.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287469+0000", 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287479+0000", 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0482000000000001e-05 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287502+0000", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2767000000000001e-05 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.287513+0000", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1012999999999999e-05 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.314525+0000", 2023-10-09T04:49:44.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027012347999999999 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.314592+0000", 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6934999999999995e-05 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.427734+0000", 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.113141357 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.427777+0000", 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3945e-05 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:151891 8.e e236/235 8:7a92d9fa:::benchmark_data_smithi082_175677_object34877:head v 236'9569, mlcod=223'2174)", 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.297945+0000", 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 285.99352090500003, 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.14090699800000001, 2023-10-09T04:49:44.317 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297945+0000", 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297945+0000", 2023-10-09T04:49:44.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297944+0000", 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297953+0000", 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6490000000000003e-06 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297970+0000", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.626e-05 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.297982+0000", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2052000000000001e-05 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.344978+0000", 2023-10-09T04:49:44.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.046996367999999997 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.345010+0000", 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1637000000000002e-05 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438805+0000", 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.093795370000000003 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438852+0000", 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6845999999999998e-05 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:151900 8.e e236/235 8:7ca9b800:::benchmark_data_smithi082_175677_object34886:head v 236'9570, mlcod=223'2174)", 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.300324+0000", 2023-10-09T04:49:44.322 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 285.99114195300001, 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.13855610300000001, 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300324+0000", 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300324+0000", 2023-10-09T04:49:44.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300323+0000", 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300337+0000", 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4187e-05 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300352+0000", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4450000000000001e-05 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.300361+0000", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2299999999999997e-06 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.356748+0000", 2023-10-09T04:49:44.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.056386868 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.356776+0000", 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8631000000000001e-05 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438855+0000", 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.082079040000000006 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438880+0000", 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4793000000000001e-05 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:151905 8.e e236/235 8:77fc4e0b:::benchmark_data_smithi082_175677_object34891:head v 236'9571, mlcod=223'2174)", 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.310871+0000", 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 285.98059504600002, 2023-10-09T04:49:44.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.128031013, 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310871+0000", 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310871+0000", 2023-10-09T04:49:44.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310870+0000", 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310881+0000", 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1415e-05 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310901+0000", 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9848e-05 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.310912+0000", 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1344e-05 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.394825+0000", 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.083912516000000006 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.394854+0000", 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9470000000000001e-05 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438882+0000", 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.044027944999999999 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.438902+0000", 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9519999999999999e-05 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "MOSDPGPush(8.e 236/235 [PushOp(8:79c1be36:::benchmark_data_smithi082_175677_object34911:head, version: 223'2194, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:79c1be36:::benchmark_data_smithi082_175677_object34911:head@223'2194, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:44:58.352156+0000", 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 285.939309761, 2023-10-09T04:49:44.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.119919282, 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352156+0000", 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352156+0000", 2023-10-09T04:49:44.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352155+0000", 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352243+0000", 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8595000000000001e-05 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352267+0000", 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3281999999999998e-05 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.352288+0000", 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0857999999999999e-05 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.469116+0000", 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11682788500000001 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.469128+0000", 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2978e-05 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:44:58.472075+0000", 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0029469660000000001 2023-10-09T04:49:44.336 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:155801 8.d e237/235 8:b2b37d75:::benchmark_data_smithi082_175677_object38787:head v 237'9860, mlcod=223'2037)", 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:04.868930+0000", 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 279.42253607800001, 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10783177200000001, 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868930+0000", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868930+0000", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868929+0000", 2023-10-09T04:49:44.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868941+0000", 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2102e-05 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868962+0000", 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1537999999999999e-05 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.868973+0000", 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1106e-05 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.883320+0000", 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.014346925999999999 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.883357+0000", 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6687999999999998e-05 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.976640+0000", 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.093283497000000007 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:04.976762+0000", 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000121209 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.341 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:161298 8.4 e237/235 8:2ae2ed88:::benchmark_data_smithi082_175677_object44284:head v 237'10063, mlcod=223'1947)", 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:13.965404+0000", 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 270.32606169899998, 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.13527583700000001, 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.342 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965404+0000", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965404+0000", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965403+0000", 2023-10-09T04:49:44.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965412+0000", 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1260000000000004e-06 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965424+0000", 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1351e-05 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.965433+0000", 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9290000000000004e-06 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.988670+0000", 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023237422000000001 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.988693+0000", 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3408e-05 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100633+0000", 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.11193916199999999 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100680+0000", 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7407000000000002e-05 2023-10-09T04:49:44.346 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:161309 8.4 e237/235 8:2895cd5e:::benchmark_data_smithi082_175677_object44295:head v 237'10064, mlcod=223'1947)", 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:13.978554+0000", 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 270.31291152, 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12215332800000001, 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978554+0000", 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978554+0000", 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978554+0000", 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978558+0000", 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7610000000000001e-06 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978567+0000", 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1689999999999998e-06 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978571+0000", 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0280000000000002e-06 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.005744+0000", 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027173025999999999 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.005774+0000", 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9947999999999999e-05 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100684+0000", 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.094909550999999995 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100708+0000", 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4192000000000001e-05 2023-10-09T04:49:44.351 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:161311 8.4 e237/235 8:28a9ec27:::benchmark_data_smithi082_175677_object44297:head v 237'10065, mlcod=223'1947)", 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:13.978575+0000", 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 270.31289055799999, 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12215804099999999, 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978575+0000", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978575+0000", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978575+0000", 2023-10-09T04:49:44.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967296 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978580+0000", 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4309999999999996e-06 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978584+0000", 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2649999999999998e-06 2023-10-09T04:49:44.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.978587+0000", 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3220000000000001e-06 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.016990+0000", 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.038402533000000003 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.017019+0000", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9201999999999999e-05 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100713+0000", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.083693848000000001 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100733+0000", 2023-10-09T04:49:44.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0598000000000002e-05 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:161313 8.4 e237/235 8:2916e78c:::benchmark_data_smithi082_175677_object44299:head v 237'10066, mlcod=223'1947)", 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:13.986826+0000", 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 270.30464013900001, 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.113936441, 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.357 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986826+0000", 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986826+0000", 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986825+0000", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986834+0000", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8189999999999998e-06 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986845+0000", 2023-10-09T04:49:44.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1197e-05 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:13.986853+0000", 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5120000000000008e-06 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.028294+0000", 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.041440420999999998 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.028343+0000", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9165000000000001e-05 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100735+0000", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.072392551999999999 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:14.100762+0000", 2023-10-09T04:49:44.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6828000000000001e-05 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:188195 8.4 e240/235 8:2563d0fc:::benchmark_data_smithi082_175677_object71181:head v 240'11743, mlcod=223'1947)", 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:45:58.767980+0000", 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 225.523486179, 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10898313900000001, 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.362 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.767980+0000", 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.767980+0000", 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.767978+0000", 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.767989+0000", 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0949e-05 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.768004+0000", 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4364e-05 2023-10-09T04:49:44.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.768011+0000", 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8990000000000004e-06 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.770929+0000", 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0029184150000000002 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.770965+0000", 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5861000000000001e-05 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.876882+0000", 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.10591726899999999 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:45:58.876963+0000", 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0653000000000002e-05 2023-10-09T04:49:44.366 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:199466 8.4 e241/235 8:2e4f39d6:::benchmark_data_smithi082_175677_object82452:head v 241'12466, mlcod=223'1947)", 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:46:17.478452+0000", 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 206.813014194, 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.112214598, 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478452+0000", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478452+0000", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478451+0000", 2023-10-09T04:49:44.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478455+0000", 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6650000000000001e-06 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478463+0000", 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3319999999999992e-06 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.478468+0000", 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4270000000000001e-06 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.497987+0000", 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019518691000000001 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.498012+0000", 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5233e-05 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.590630+0000", 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.092618421000000006 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:46:17.590666+0000", 2023-10-09T04:49:44.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6106000000000003e-05 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:225618 8.4 e245/235 8:2ae09b4c:::benchmark_data_smithi082_175677_object108604:head v 245'14110, mlcod=223'1947)", 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:47:00.281532+0000", 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 164.00993362700001, 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.111655186, 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281532+0000", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281532+0000", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281531+0000", 2023-10-09T04:49:44.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281544+0000", 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3018e-05 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281558+0000", 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3360000000000001e-05 2023-10-09T04:49:44.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.281565+0000", 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6669999999999996e-06 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.308669+0000", 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027103267 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.308692+0000", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3819999999999999e-05 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.393150+0000", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.084457381999999998 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:00.393187+0000", 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7645000000000003e-05 2023-10-09T04:49:44.376 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:227176 8.d e245/235 8:b72e031a:::benchmark_data_smithi082_175677_object110162:head v 245'14298, mlcod=223'2037)", 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:47:02.815724+0000", 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 161.47574217799999, 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.108178582, 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.377 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815724+0000", 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815724+0000", 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815723+0000", 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815732+0000", 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4709999999999996e-06 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815747+0000", 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4691999999999999e-05 2023-10-09T04:49:44.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.815755+0000", 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3310000000000004e-06 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.843807+0000", 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.028052123000000002 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.843839+0000", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1856999999999997e-05 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.923674+0000", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.079835027000000003 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:02.923902+0000", 2023-10-09T04:49:44.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00022814999999999999 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:228278 8.4 e245/235 8:284b81f6:::benchmark_data_smithi082_175677_object111264:head v 245'14275, mlcod=223'1947)", 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:47:04.602182+0000", 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 159.68928376599999, 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12090970700000001, 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.382 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602182+0000", 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602182+0000", 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.383 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602182+0000", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602187+0000", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9649999999999999e-06 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602194+0000", 2023-10-09T04:49:44.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0190000000000001e-06 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.602198+0000", 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.647747+0000", 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.045548472 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.647802+0000", 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4675000000000001e-05 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.723054+0000", 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.075252101000000002 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.723092+0000", 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8207000000000002e-05 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:228281 8.4 e245/235 8:26dae3a9:::benchmark_data_smithi082_175677_object111267:head v 245'14276, mlcod=223'1947)", 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:47:04.609825+0000", 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 159.681641423, 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.12300841, 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.387 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609825+0000", 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609825+0000", 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609823+0000", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609834+0000", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0726e-05 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609853+0000", 2023-10-09T04:49:44.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9415e-05 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.609863+0000", 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6069999999999997e-06 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.666233+0000", 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.056369996999999998 2023-10-09T04:49:44.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.666260+0000", 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7022e-05 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.732762+0000", 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.066501665000000001 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.732833+0000", 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1378999999999995e-05 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35442.0:228287 8.4 e245/235 8:29d00592:::benchmark_data_smithi082_175677_object111273:head v 245'14277, mlcod=223'1947)", 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:47:04.617248+0000", 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 159.67421838999999, 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.115647137, 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:44.392 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617248+0000", 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617248+0000", 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:49:44.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617247+0000", 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617252+0000", 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3869999999999997e-06 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617259+0000", 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2790000000000001e-06 2023-10-09T04:49:44.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.617263+0000", 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.442e-06 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.678223+0000", 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.060959603000000001 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.678243+0000", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0395e-05 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.732839+0000", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.054595861000000002 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:47:04.732895+0000", 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5491000000000001e-05 2023-10-09T04:49:44.396 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.397 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.397 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.397 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:49:44.397 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:49:44.397 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:49:44.398 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:44 smithi082 bash[77063]: cluster 2023-10-09T04:49:42.735942+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T04:49:44.398 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:44 smithi153 bash[59565]: cluster 2023-10-09T04:49:42.735942+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T04:49:44.399 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:44 smithi082 bash[79733]: cluster 2023-10-09T04:49:42.735942+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T04:49:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:44.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:44.486 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1140,"stamp":"2023-10-09T04:49:40.734951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6186735539,"num_objects":94357,"num_object_clones":0,"num_object_copies":283071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94357,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":95435,"num_write_kb":6052215,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8398292,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95499,"ondisk_log_size":95499,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22522760,"kb_used_data":18112200,"kb_used_omap":47,"kb_used_meta":4410448,"kb_avail":727438456,"statfs":{"total":767960285184,"available":744896978944,"internally_reserved":0,"allocated":18546892800,"data_stored":18540535310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":4516299165},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,3,5,4,2,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":815857664,"num_objects":12449,"num_object_clones":0,"num_object_copies":37347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12449,"num_write_kb":796736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004315"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":251,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079144+0000","last_change":"2023-10-09T04:44:51.896564+0000","last_active":"2023-10-09T04:48:54.079144+0000","last_peered":"2023-10-09T04:48:54.079144+0000","last_clean":"2023-10-09T04:48:54.079144+0000","last_became_active":"2023-10-09T04:44:51.896195+0000","last_became_peered":"2023-10-09T04:44:51.896195+0000","last_unstale":"2023-10-09T04:48:54.079144+0000","last_undegraded":"2023-10-09T04:48:54.079144+0000","last_fullsized":"2023-10-09T04:48:54.079144+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":210,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.097773+0000","last_change":"2023-10-09T04:44:51.891799+0000","last_active":"2023-10-09T04:48:54.097773+0000","last_peered":"2023-10-09T04:48:54.097773+0000","last_clean":"2023-10-09T04:48:54.097773+0000","last_became_active":"2023-10-09T04:44:51.891479+0000","last_became_peered":"2023-10-09T04:44:51.891479+0000","last_unstale":"2023-10-09T04:48:54.097773+0000","last_undegraded":"2023-10-09T04:48:54.097773+0000","last_fullsized":"2023-10-09T04:48:54.097773+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:57.730414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":260,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.079731+0000","last_change":"2023-10-09T04:43:53.989477+0000","last_active":"2023-10-09T04:49:19.079731+0000","last_peered":"2023-10-09T04:49:19.079731+0000","last_clean":"2023-10-09T04:49:19.079731+0000","last_became_active":"2023-10-09T04:43:53.988910+0000","last_became_peered":"2023-10-09T04:43:53.988910+0000","last_unstale":"2023-10-09T04:49:19.079731+0000","last_undegraded":"2023-10-09T04:49:19.079731+0000","last_fullsized":"2023-10-09T04:49:19.079731+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:35.144801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":233,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079763+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:48:54.079763+0000","last_peered":"2023-10-09T04:48:54.079763+0000","last_clean":"2023-10-09T04:48:54.079763+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:48:54.079763+0000","last_undegraded":"2023-10-09T04:48:54.079763+0000","last_fullsized":"2023-10-09T04:48:54.079763+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079417+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:48:54.079417+0000","last_peered":"2023-10-09T04:48:54.079417+0000","last_clean":"2023-10-09T04:48:54.079417+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:48:54.079417+0000","last_undegraded":"2023-10-09T04:48:54.079417+0000","last_fullsized":"2023-10-09T04:48:54.079417+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":210,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093590+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:48:54.093590+0000","last_peered":"2023-10-09T04:48:54.093590+0000","last_clean":"2023-10-09T04:48:54.093590+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:48:54.093590+0000","last_undegraded":"2023-10-09T04:48:54.093590+0000","last_fullsized":"2023-10-09T04:48:54.093590+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":228,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:59.319384+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:48:59.319384+0000","last_peered":"2023-10-09T04:48:59.319384+0000","last_clean":"2023-10-09T04:48:59.319384+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:48:59.319384+0000","last_undegraded":"2023-10-09T04:48:59.319384+0000","last_fullsized":"2023-10-09T04:48:59.319384+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":176,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.082315+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:48:54.082315+0000","last_peered":"2023-10-09T04:48:54.082315+0000","last_clean":"2023-10-09T04:48:54.082315+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:48:54.082315+0000","last_undegraded":"2023-10-09T04:48:54.082315+0000","last_fullsized":"2023-10-09T04:48:54.082315+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":196,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088528+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:48:54.088528+0000","last_peered":"2023-10-09T04:48:54.088528+0000","last_clean":"2023-10-09T04:48:54.088528+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:48:54.088528+0000","last_undegraded":"2023-10-09T04:48:54.088528+0000","last_fullsized":"2023-10-09T04:48:54.088528+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084922+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:48:54.084922+0000","last_peered":"2023-10-09T04:48:54.084922+0000","last_clean":"2023-10-09T04:48:54.084922+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:48:54.084922+0000","last_undegraded":"2023-10-09T04:48:54.084922+0000","last_fullsized":"2023-10-09T04:48:54.084922+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":76,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084484+0000","last_change":"2023-10-09T04:40:47.996370+0000","last_active":"2023-10-09T04:48:54.084484+0000","last_peered":"2023-10-09T04:48:54.084484+0000","last_clean":"2023-10-09T04:48:54.084484+0000","last_became_active":"2023-10-09T04:40:47.817084+0000","last_became_peered":"2023-10-09T04:40:47.817084+0000","last_unstale":"2023-10-09T04:48:54.084484+0000","last_undegraded":"2023-10-09T04:48:54.084484+0000","last_fullsized":"2023-10-09T04:48:54.084484+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":239,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093967+0000","last_change":"2023-10-09T04:44:51.903464+0000","last_active":"2023-10-09T04:48:54.093967+0000","last_peered":"2023-10-09T04:48:54.093967+0000","last_clean":"2023-10-09T04:48:54.093967+0000","last_became_active":"2023-10-09T04:44:51.903078+0000","last_became_peered":"2023-10-09T04:44:51.903078+0000","last_unstale":"2023-10-09T04:48:54.093967+0000","last_undegraded":"2023-10-09T04:48:54.093967+0000","last_fullsized":"2023-10-09T04:48:54.093967+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:57.992171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":280,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079582+0000","last_change":"2023-10-09T04:40:47.863039+0000","last_active":"2023-10-09T04:48:54.079582+0000","last_peered":"2023-10-09T04:48:54.079582+0000","last_clean":"2023-10-09T04:48:54.079582+0000","last_became_active":"2023-10-09T04:40:47.819279+0000","last_became_peered":"2023-10-09T04:40:47.819279+0000","last_unstale":"2023-10-09T04:48:54.079582+0000","last_undegraded":"2023-10-09T04:48:54.079582+0000","last_fullsized":"2023-10-09T04:48:54.079582+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.080400+0000","last_change":"2023-10-09T04:34:09.911494+0000","last_active":"2023-10-09T04:48:54.080400+0000","last_peered":"2023-10-09T04:48:54.080400+0000","last_clean":"2023-10-09T04:48:54.080400+0000","last_became_active":"2023-10-09T04:34:09.910917+0000","last_became_peered":"2023-10-09T04:34:09.910917+0000","last_unstale":"2023-10-09T04:48:54.080400+0000","last_undegraded":"2023-10-09T04:48:54.080400+0000","last_fullsized":"2023-10-09T04:48:54.080400+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":91,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084667+0000","last_change":"2023-10-09T04:44:51.899516+0000","last_active":"2023-10-09T04:48:54.084667+0000","last_peered":"2023-10-09T04:48:54.084667+0000","last_clean":"2023-10-09T04:48:54.084667+0000","last_became_active":"2023-10-09T04:44:51.899092+0000","last_became_peered":"2023-10-09T04:44:51.899092+0000","last_unstale":"2023-10-09T04:48:54.084667+0000","last_undegraded":"2023-10-09T04:48:54.084667+0000","last_fullsized":"2023-10-09T04:48:54.084667+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.090817+0000","last_change":"2023-10-09T04:40:47.816775+0000","last_active":"2023-10-09T04:48:54.090817+0000","last_peered":"2023-10-09T04:48:54.090817+0000","last_clean":"2023-10-09T04:48:54.090817+0000","last_became_active":"2023-10-09T04:40:47.816424+0000","last_became_peered":"2023-10-09T04:40:47.816424+0000","last_unstale":"2023-10-09T04:48:54.090817+0000","last_undegraded":"2023-10-09T04:48:54.090817+0000","last_fullsized":"2023-10-09T04:48:54.090817+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:52.923977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":231,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079003+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:48:54.079003+0000","last_peered":"2023-10-09T04:48:54.079003+0000","last_clean":"2023-10-09T04:48:54.079003+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:48:54.079003+0000","last_undegraded":"2023-10-09T04:48:54.079003+0000","last_fullsized":"2023-10-09T04:48:54.079003+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088020+0000","last_change":"2023-10-09T04:40:47.820893+0000","last_active":"2023-10-09T04:48:54.088020+0000","last_peered":"2023-10-09T04:48:54.088020+0000","last_clean":"2023-10-09T04:48:54.088020+0000","last_became_active":"2023-10-09T04:40:47.820570+0000","last_became_peered":"2023-10-09T04:40:47.820570+0000","last_unstale":"2023-10-09T04:48:54.088020+0000","last_undegraded":"2023-10-09T04:48:54.088020+0000","last_fullsized":"2023-10-09T04:48:54.088020+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084969+0000","last_change":"2023-10-09T04:40:47.823962+0000","last_active":"2023-10-09T04:48:54.084969+0000","last_peered":"2023-10-09T04:48:54.084969+0000","last_clean":"2023-10-09T04:48:54.084969+0000","last_became_active":"2023-10-09T04:40:47.823509+0000","last_became_peered":"2023-10-09T04:40:47.823509+0000","last_unstale":"2023-10-09T04:48:54.084969+0000","last_undegraded":"2023-10-09T04:48:54.084969+0000","last_fullsized":"2023-10-09T04:48:54.084969+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":231,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:59.319367+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:48:59.319367+0000","last_peered":"2023-10-09T04:48:59.319367+0000","last_clean":"2023-10-09T04:48:59.319367+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:48:59.319367+0000","last_undegraded":"2023-10-09T04:48:59.319367+0000","last_fullsized":"2023-10-09T04:48:59.319367+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":219,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093204+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:48:54.093204+0000","last_peered":"2023-10-09T04:48:54.093204+0000","last_clean":"2023-10-09T04:48:54.093204+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:48:54.093204+0000","last_undegraded":"2023-10-09T04:48:54.093204+0000","last_fullsized":"2023-10-09T04:48:54.093204+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.083099+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:48:54.083099+0000","last_peered":"2023-10-09T04:48:54.083099+0000","last_clean":"2023-10-09T04:48:54.083099+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:48:54.083099+0000","last_undegraded":"2023-10-09T04:48:54.083099+0000","last_fullsized":"2023-10-09T04:48:54.083099+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"255'6019","reported_seq":6033,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.448207+0000","last_change":"2023-10-09T04:48:02.044407+0000","last_active":"2023-10-09T04:49:39.448207+0000","last_peered":"2023-10-09T04:49:39.448207+0000","last_clean":"2023-10-09T04:49:39.448207+0000","last_became_active":"2023-10-09T04:48:02.043959+0000","last_became_peered":"2023-10-09T04:48:02.043959+0000","last_unstale":"2023-10-09T04:49:39.448207+0000","last_undegraded":"2023-10-09T04:49:39.448207+0000","last_fullsized":"2023-10-09T04:49:39.448207+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:02.677050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394461184,"num_objects":6019,"num_object_clones":0,"num_object_copies":18057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6019,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":74,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.086383+0000","last_change":"2023-10-09T04:40:48.230655+0000","last_active":"2023-10-09T04:48:54.086383+0000","last_peered":"2023-10-09T04:48:54.086383+0000","last_clean":"2023-10-09T04:48:54.086383+0000","last_became_active":"2023-10-09T04:40:47.830826+0000","last_became_peered":"2023-10-09T04:40:47.830826+0000","last_unstale":"2023-10-09T04:48:54.086383+0000","last_undegraded":"2023-10-09T04:48:54.086383+0000","last_fullsized":"2023-10-09T04:48:54.086383+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":187,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.087720+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:48:54.087720+0000","last_peered":"2023-10-09T04:48:54.087720+0000","last_clean":"2023-10-09T04:48:54.087720+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:48:54.087720+0000","last_undegraded":"2023-10-09T04:48:54.087720+0000","last_fullsized":"2023-10-09T04:48:54.087720+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"255'5920","reported_seq":5932,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.837796+0000","last_change":"2023-10-09T04:48:02.139233+0000","last_active":"2023-10-09T04:49:39.837796+0000","last_peered":"2023-10-09T04:49:39.837796+0000","last_clean":"2023-10-09T04:49:39.837796+0000","last_became_active":"2023-10-09T04:48:02.138632+0000","last_became_peered":"2023-10-09T04:48:02.138632+0000","last_unstale":"2023-10-09T04:49:39.837796+0000","last_undegraded":"2023-10-09T04:49:39.837796+0000","last_fullsized":"2023-10-09T04:49:39.837796+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:52.944610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387973120,"num_objects":5920,"num_object_clones":0,"num_object_copies":17760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5920,"num_write_kb":378880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":87,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084808+0000","last_change":"2023-10-09T04:43:54.055892+0000","last_active":"2023-10-09T04:48:54.084808+0000","last_peered":"2023-10-09T04:48:54.084808+0000","last_clean":"2023-10-09T04:48:54.084808+0000","last_became_active":"2023-10-09T04:43:53.987210+0000","last_became_peered":"2023-10-09T04:43:53.987210+0000","last_unstale":"2023-10-09T04:48:54.084808+0000","last_undegraded":"2023-10-09T04:48:54.084808+0000","last_fullsized":"2023-10-09T04:48:54.084808+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079394+0000","last_change":"2023-10-09T04:24:00.827553+0000","last_active":"2023-10-09T04:48:54.079394+0000","last_peered":"2023-10-09T04:48:54.079394+0000","last_clean":"2023-10-09T04:48:54.079394+0000","last_became_active":"2023-10-09T04:24:00.827210+0000","last_became_peered":"2023-10-09T04:24:00.827210+0000","last_unstale":"2023-10-09T04:48:54.079394+0000","last_undegraded":"2023-10-09T04:48:54.079394+0000","last_fullsized":"2023-10-09T04:48:54.079394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"255'5899","reported_seq":5912,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.638227+0000","last_change":"2023-10-09T04:48:02.157308+0000","last_active":"2023-10-09T04:49:39.638227+0000","last_peered":"2023-10-09T04:49:39.638227+0000","last_clean":"2023-10-09T04:49:39.638227+0000","last_became_active":"2023-10-09T04:48:02.156513+0000","last_became_peered":"2023-10-09T04:48:02.156513+0000","last_unstale":"2023-10-09T04:49:39.638227+0000","last_undegraded":"2023-10-09T04:49:39.638227+0000","last_fullsized":"2023-10-09T04:49:39.638227+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386596864,"num_objects":5899,"num_object_clones":0,"num_object_copies":17697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5899,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":74,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084611+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:48:54.084611+0000","last_peered":"2023-10-09T04:48:54.084611+0000","last_clean":"2023-10-09T04:48:54.084611+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:48:54.084611+0000","last_undegraded":"2023-10-09T04:48:54.084611+0000","last_fullsized":"2023-10-09T04:48:54.084611+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":239,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093359+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:48:54.093359+0000","last_peered":"2023-10-09T04:48:54.093359+0000","last_clean":"2023-10-09T04:48:54.093359+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:48:54.093359+0000","last_undegraded":"2023-10-09T04:48:54.093359+0000","last_fullsized":"2023-10-09T04:48:54.093359+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"255'5864","reported_seq":5877,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.644397+0000","last_change":"2023-10-09T04:48:02.045217+0000","last_active":"2023-10-09T04:49:39.644397+0000","last_peered":"2023-10-09T04:49:39.644397+0000","last_clean":"2023-10-09T04:49:39.644397+0000","last_became_active":"2023-10-09T04:48:02.044927+0000","last_became_peered":"2023-10-09T04:48:02.044927+0000","last_unstale":"2023-10-09T04:49:39.644397+0000","last_undegraded":"2023-10-09T04:49:39.644397+0000","last_fullsized":"2023-10-09T04:49:39.644397+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384303104,"num_objects":5864,"num_object_clones":0,"num_object_copies":17592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5864,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.087700+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:48:54.087700+0000","last_peered":"2023-10-09T04:48:54.087700+0000","last_clean":"2023-10-09T04:48:54.087700+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:48:54.087700+0000","last_undegraded":"2023-10-09T04:48:54.087700+0000","last_fullsized":"2023-10-09T04:48:54.087700+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":242,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.061634+0000","last_change":"2023-10-09T04:44:51.892263+0000","last_active":"2023-10-09T04:49:19.061634+0000","last_peered":"2023-10-09T04:49:19.061634+0000","last_clean":"2023-10-09T04:49:19.061634+0000","last_became_active":"2023-10-09T04:44:51.891917+0000","last_became_peered":"2023-10-09T04:44:51.891917+0000","last_unstale":"2023-10-09T04:49:19.061634+0000","last_undegraded":"2023-10-09T04:49:19.061634+0000","last_fullsized":"2023-10-09T04:49:19.061634+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:26.796713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"255'5838","reported_seq":5851,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:38.486933+0000","last_change":"2023-10-09T04:48:02.156998+0000","last_active":"2023-10-09T04:49:38.486933+0000","last_peered":"2023-10-09T04:49:38.486933+0000","last_clean":"2023-10-09T04:49:38.486933+0000","last_became_active":"2023-10-09T04:48:02.156119+0000","last_became_peered":"2023-10-09T04:48:02.156119+0000","last_unstale":"2023-10-09T04:49:38.486933+0000","last_undegraded":"2023-10-09T04:49:38.486933+0000","last_fullsized":"2023-10-09T04:49:38.486933+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382599168,"num_objects":5838,"num_object_clones":0,"num_object_copies":17514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5838,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":177,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.321787+0000","last_change":"2023-10-09T04:28:51.530688+0000","last_active":"2023-10-09T04:49:04.321787+0000","last_peered":"2023-10-09T04:49:04.321787+0000","last_clean":"2023-10-09T04:49:04.321787+0000","last_became_active":"2023-10-09T04:28:51.530291+0000","last_became_peered":"2023-10-09T04:28:51.530291+0000","last_unstale":"2023-10-09T04:49:04.321787+0000","last_undegraded":"2023-10-09T04:49:04.321787+0000","last_fullsized":"2023-10-09T04:49:04.321787+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:13.309605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079660+0000","last_change":"2023-10-09T04:34:09.967932+0000","last_active":"2023-10-09T04:48:54.079660+0000","last_peered":"2023-10-09T04:48:54.079660+0000","last_clean":"2023-10-09T04:48:54.079660+0000","last_became_active":"2023-10-09T04:34:09.967590+0000","last_became_peered":"2023-10-09T04:34:09.967590+0000","last_unstale":"2023-10-09T04:48:54.079660+0000","last_undegraded":"2023-10-09T04:48:54.079660+0000","last_fullsized":"2023-10-09T04:48:54.079660+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"255'5981","reported_seq":5995,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.822166+0000","last_change":"2023-10-09T04:48:02.036963+0000","last_active":"2023-10-09T04:49:39.822166+0000","last_peered":"2023-10-09T04:49:39.822166+0000","last_clean":"2023-10-09T04:49:39.822166+0000","last_became_active":"2023-10-09T04:48:02.036549+0000","last_became_peered":"2023-10-09T04:48:02.036549+0000","last_unstale":"2023-10-09T04:49:39.822166+0000","last_undegraded":"2023-10-09T04:49:39.822166+0000","last_fullsized":"2023-10-09T04:49:39.822166+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5981,"log_dups_size":0,"ondisk_log_size":5981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.195147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391970816,"num_objects":5981,"num_object_clones":0,"num_object_copies":17943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5981,"num_write_kb":382784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093267+0000","last_change":"2023-10-09T04:34:09.982729+0000","last_active":"2023-10-09T04:48:54.093267+0000","last_peered":"2023-10-09T04:48:54.093267+0000","last_clean":"2023-10-09T04:48:54.093267+0000","last_became_active":"2023-10-09T04:34:09.974819+0000","last_became_peered":"2023-10-09T04:34:09.974819+0000","last_unstale":"2023-10-09T04:48:54.093267+0000","last_undegraded":"2023-10-09T04:48:54.093267+0000","last_fullsized":"2023-10-09T04:48:54.093267+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":48,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079070+0000","last_change":"2023-10-09T04:44:53.333649+0000","last_active":"2023-10-09T04:48:54.079070+0000","last_peered":"2023-10-09T04:48:54.079070+0000","last_clean":"2023-10-09T04:48:54.079070+0000","last_became_active":"2023-10-09T04:44:53.332464+0000","last_became_peered":"2023-10-09T04:44:53.332464+0000","last_unstale":"2023-10-09T04:48:54.079070+0000","last_undegraded":"2023-10-09T04:48:54.079070+0000","last_fullsized":"2023-10-09T04:48:54.079070+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"255'5772","reported_seq":5785,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:35.941807+0000","last_change":"2023-10-09T04:48:02.140445+0000","last_active":"2023-10-09T04:49:35.941807+0000","last_peered":"2023-10-09T04:49:35.941807+0000","last_clean":"2023-10-09T04:49:35.941807+0000","last_became_active":"2023-10-09T04:48:02.139406+0000","last_became_peered":"2023-10-09T04:48:02.139406+0000","last_unstale":"2023-10-09T04:49:35.941807+0000","last_undegraded":"2023-10-09T04:49:35.941807+0000","last_fullsized":"2023-10-09T04:49:35.941807+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378273792,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":369408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079142+0000","last_change":"2023-10-09T04:44:53.102191+0000","last_active":"2023-10-09T04:48:54.079142+0000","last_peered":"2023-10-09T04:48:54.079142+0000","last_clean":"2023-10-09T04:48:54.079142+0000","last_became_active":"2023-10-09T04:44:53.101206+0000","last_became_peered":"2023-10-09T04:44:53.101206+0000","last_unstale":"2023-10-09T04:48:54.079142+0000","last_undegraded":"2023-10-09T04:48:54.079142+0000","last_fullsized":"2023-10-09T04:48:54.079142+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079990+0000","last_change":"2023-10-09T04:34:09.916419+0000","last_active":"2023-10-09T04:48:54.079990+0000","last_peered":"2023-10-09T04:48:54.079990+0000","last_clean":"2023-10-09T04:48:54.079990+0000","last_became_active":"2023-10-09T04:34:09.916035+0000","last_became_peered":"2023-10-09T04:34:09.916035+0000","last_unstale":"2023-10-09T04:48:54.079990+0000","last_undegraded":"2023-10-09T04:48:54.079990+0000","last_fullsized":"2023-10-09T04:48:54.079990+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"255'5992","reported_seq":6006,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.798515+0000","last_change":"2023-10-09T04:48:02.139290+0000","last_active":"2023-10-09T04:49:39.798515+0000","last_peered":"2023-10-09T04:49:39.798515+0000","last_clean":"2023-10-09T04:49:39.798515+0000","last_became_active":"2023-10-09T04:48:02.138631+0000","last_became_peered":"2023-10-09T04:48:02.138631+0000","last_unstale":"2023-10-09T04:49:39.798515+0000","last_undegraded":"2023-10-09T04:49:39.798515+0000","last_fullsized":"2023-10-09T04:49:39.798515+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5992,"log_dups_size":0,"ondisk_log_size":5992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:32.990209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392691712,"num_objects":5992,"num_object_clones":0,"num_object_copies":17976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5992,"num_write_kb":383488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.082073+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:48:54.082073+0000","last_peered":"2023-10-09T04:48:54.082073+0000","last_clean":"2023-10-09T04:48:54.082073+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:48:54.082073+0000","last_undegraded":"2023-10-09T04:48:54.082073+0000","last_fullsized":"2023-10-09T04:48:54.082073+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":209,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.090440+0000","last_change":"2023-10-09T04:24:00.825456+0000","last_active":"2023-10-09T04:48:54.090440+0000","last_peered":"2023-10-09T04:48:54.090440+0000","last_clean":"2023-10-09T04:48:54.090440+0000","last_became_active":"2023-10-09T04:24:00.824960+0000","last_became_peered":"2023-10-09T04:24:00.824960+0000","last_unstale":"2023-10-09T04:48:54.090440+0000","last_undegraded":"2023-10-09T04:48:54.090440+0000","last_fullsized":"2023-10-09T04:48:54.090440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:55.167143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"255'6006","reported_seq":6019,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.621235+0000","last_change":"2023-10-09T04:48:02.156639+0000","last_active":"2023-10-09T04:49:39.621235+0000","last_peered":"2023-10-09T04:49:39.621235+0000","last_clean":"2023-10-09T04:49:39.621235+0000","last_became_active":"2023-10-09T04:48:02.155893+0000","last_became_peered":"2023-10-09T04:48:02.155893+0000","last_unstale":"2023-10-09T04:49:39.621235+0000","last_undegraded":"2023-10-09T04:49:39.621235+0000","last_fullsized":"2023-10-09T04:49:39.621235+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393609216,"num_objects":6006,"num_object_clones":0,"num_object_copies":18018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6006,"num_write_kb":384384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":709,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093289+0000","last_change":"2023-10-09T04:41:04.902111+0000","last_active":"2023-10-09T04:48:54.093289+0000","last_peered":"2023-10-09T04:48:54.093289+0000","last_clean":"2023-10-09T04:48:54.093289+0000","last_became_active":"2023-10-09T04:41:04.901604+0000","last_became_peered":"2023-10-09T04:41:04.901604+0000","last_unstale":"2023-10-09T04:48:54.093289+0000","last_undegraded":"2023-10-09T04:48:54.093289+0000","last_fullsized":"2023-10-09T04:48:54.093289+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:55.374183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":218,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088202+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:48:54.088202+0000","last_peered":"2023-10-09T04:48:54.088202+0000","last_clean":"2023-10-09T04:48:54.088202+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:48:54.088202+0000","last_undegraded":"2023-10-09T04:48:54.088202+0000","last_fullsized":"2023-10-09T04:48:54.088202+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":228,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.094099+0000","last_change":"2023-10-09T04:43:53.993014+0000","last_active":"2023-10-09T04:48:54.094099+0000","last_peered":"2023-10-09T04:48:54.094099+0000","last_clean":"2023-10-09T04:48:54.094099+0000","last_became_active":"2023-10-09T04:43:53.992702+0000","last_became_peered":"2023-10-09T04:43:53.992702+0000","last_unstale":"2023-10-09T04:48:54.094099+0000","last_undegraded":"2023-10-09T04:48:54.094099+0000","last_fullsized":"2023-10-09T04:48:54.094099+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:47.151693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"255'5922","reported_seq":5936,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.434479+0000","last_change":"2023-10-09T04:48:02.038776+0000","last_active":"2023-10-09T04:49:39.434479+0000","last_peered":"2023-10-09T04:49:39.434479+0000","last_clean":"2023-10-09T04:49:39.434479+0000","last_became_active":"2023-10-09T04:48:02.038418+0000","last_became_peered":"2023-10-09T04:48:02.038418+0000","last_unstale":"2023-10-09T04:49:39.434479+0000","last_undegraded":"2023-10-09T04:49:39.434479+0000","last_fullsized":"2023-10-09T04:49:39.434479+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:26.911517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104192,"num_objects":5922,"num_object_clones":0,"num_object_copies":17766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":717,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088396+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:48:54.088396+0000","last_peered":"2023-10-09T04:48:54.088396+0000","last_clean":"2023-10-09T04:48:54.088396+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:48:54.088396+0000","last_undegraded":"2023-10-09T04:48:54.088396+0000","last_fullsized":"2023-10-09T04:48:54.088396+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.080323+0000","last_change":"2023-10-09T04:34:09.911293+0000","last_active":"2023-10-09T04:48:54.080323+0000","last_peered":"2023-10-09T04:48:54.080323+0000","last_clean":"2023-10-09T04:48:54.080323+0000","last_became_active":"2023-10-09T04:34:09.910463+0000","last_became_peered":"2023-10-09T04:34:09.910463+0000","last_unstale":"2023-10-09T04:48:54.080323+0000","last_undegraded":"2023-10-09T04:48:54.080323+0000","last_fullsized":"2023-10-09T04:48:54.080323+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.090933+0000","last_change":"2023-10-09T04:44:51.896804+0000","last_active":"2023-10-09T04:48:54.090933+0000","last_peered":"2023-10-09T04:48:54.090933+0000","last_clean":"2023-10-09T04:48:54.090933+0000","last_became_active":"2023-10-09T04:44:51.896434+0000","last_became_peered":"2023-10-09T04:44:51.896434+0000","last_unstale":"2023-10-09T04:48:54.090933+0000","last_undegraded":"2023-10-09T04:48:54.090933+0000","last_fullsized":"2023-10-09T04:48:54.090933+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:39.726149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"255'6041","reported_seq":6054,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.660239+0000","last_change":"2023-10-09T04:48:02.046283+0000","last_active":"2023-10-09T04:49:39.660239+0000","last_peered":"2023-10-09T04:49:39.660239+0000","last_clean":"2023-10-09T04:49:39.660239+0000","last_became_active":"2023-10-09T04:48:02.045795+0000","last_became_peered":"2023-10-09T04:48:02.045795+0000","last_unstale":"2023-10-09T04:49:39.660239+0000","last_undegraded":"2023-10-09T04:49:39.660239+0000","last_fullsized":"2023-10-09T04:49:39.660239+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395902976,"num_objects":6041,"num_object_clones":0,"num_object_copies":18123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6041,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1333,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:49:00.434392+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:49:00.434392+0000","last_peered":"2023-10-09T04:49:00.434392+0000","last_clean":"2023-10-09T04:49:00.434392+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:49:00.434392+0000","last_undegraded":"2023-10-09T04:49:00.434392+0000","last_fullsized":"2023-10-09T04:49:00.434392+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.759592+0000","last_change":"2023-10-09T04:44:51.903679+0000","last_active":"2023-10-09T04:48:54.759592+0000","last_peered":"2023-10-09T04:48:54.759592+0000","last_clean":"2023-10-09T04:48:54.759592+0000","last_became_active":"2023-10-09T04:44:51.903243+0000","last_became_peered":"2023-10-09T04:44:51.903243+0000","last_unstale":"2023-10-09T04:48:54.759592+0000","last_undegraded":"2023-10-09T04:48:54.759592+0000","last_fullsized":"2023-10-09T04:48:54.759592+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:35.333537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":255,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.090523+0000","last_change":"2023-10-09T04:34:36.644620+0000","last_active":"2023-10-09T04:48:54.090523+0000","last_peered":"2023-10-09T04:48:54.090523+0000","last_clean":"2023-10-09T04:48:54.090523+0000","last_became_active":"2023-10-09T04:34:36.644329+0000","last_became_peered":"2023-10-09T04:34:36.644329+0000","last_unstale":"2023-10-09T04:48:54.090523+0000","last_undegraded":"2023-10-09T04:48:54.090523+0000","last_fullsized":"2023-10-09T04:48:54.090523+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:47:15.714585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"255'5914","reported_seq":5928,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.435133+0000","last_change":"2023-10-09T04:48:02.139821+0000","last_active":"2023-10-09T04:49:39.435133+0000","last_peered":"2023-10-09T04:49:39.435133+0000","last_clean":"2023-10-09T04:49:39.435133+0000","last_became_active":"2023-10-09T04:48:02.139065+0000","last_became_peered":"2023-10-09T04:48:02.139065+0000","last_unstale":"2023-10-09T04:49:39.435133+0000","last_undegraded":"2023-10-09T04:49:39.435133+0000","last_fullsized":"2023-10-09T04:49:39.435133+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:29.788840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387579904,"num_objects":5914,"num_object_clones":0,"num_object_copies":17742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5914,"num_write_kb":378496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":780,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.087679+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:48:54.087679+0000","last_peered":"2023-10-09T04:48:54.087679+0000","last_clean":"2023-10-09T04:48:54.087679+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:48:54.087679+0000","last_undegraded":"2023-10-09T04:48:54.087679+0000","last_fullsized":"2023-10-09T04:48:54.087679+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":476,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084015+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:48:54.084015+0000","last_peered":"2023-10-09T04:48:54.084015+0000","last_clean":"2023-10-09T04:48:54.084015+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:48:54.084015+0000","last_undegraded":"2023-10-09T04:48:54.084015+0000","last_fullsized":"2023-10-09T04:48:54.084015+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":173,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088289+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:48:54.088289+0000","last_peered":"2023-10-09T04:48:54.088289+0000","last_clean":"2023-10-09T04:48:54.088289+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:48:54.088289+0000","last_undegraded":"2023-10-09T04:48:54.088289+0000","last_fullsized":"2023-10-09T04:48:54.088289+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093889+0000","last_change":"2023-10-09T04:40:47.825563+0000","last_active":"2023-10-09T04:48:54.093889+0000","last_peered":"2023-10-09T04:48:54.093889+0000","last_clean":"2023-10-09T04:48:54.093889+0000","last_became_active":"2023-10-09T04:40:47.824020+0000","last_became_peered":"2023-10-09T04:40:47.824020+0000","last_unstale":"2023-10-09T04:48:54.093889+0000","last_undegraded":"2023-10-09T04:48:54.093889+0000","last_fullsized":"2023-10-09T04:48:54.093889+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"255'5971","reported_seq":5984,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:39.442597+0000","last_change":"2023-10-09T04:48:02.042880+0000","last_active":"2023-10-09T04:49:39.442597+0000","last_peered":"2023-10-09T04:49:39.442597+0000","last_clean":"2023-10-09T04:49:39.442597+0000","last_became_active":"2023-10-09T04:48:02.042529+0000","last_became_peered":"2023-10-09T04:48:02.042529+0000","last_unstale":"2023-10-09T04:49:39.442597+0000","last_undegraded":"2023-10-09T04:49:39.442597+0000","last_fullsized":"2023-10-09T04:49:39.442597+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5971,"log_dups_size":0,"ondisk_log_size":5971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:32.083166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391315456,"num_objects":5971,"num_object_clones":0,"num_object_copies":17913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5971,"num_write_kb":382144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":642,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084023+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:48:54.084023+0000","last_peered":"2023-10-09T04:48:54.084023+0000","last_clean":"2023-10-09T04:48:54.084023+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:48:54.084023+0000","last_undegraded":"2023-10-09T04:48:54.084023+0000","last_fullsized":"2023-10-09T04:48:54.084023+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.067505+0000","last_change":"2023-10-09T04:43:53.972134+0000","last_active":"2023-10-09T04:49:19.067505+0000","last_peered":"2023-10-09T04:49:19.067505+0000","last_clean":"2023-10-09T04:49:19.067505+0000","last_became_active":"2023-10-09T04:43:53.971710+0000","last_became_peered":"2023-10-09T04:43:53.971710+0000","last_unstale":"2023-10-09T04:49:19.067505+0000","last_undegraded":"2023-10-09T04:49:19.067505+0000","last_fullsized":"2023-10-09T04:49:19.067505+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:56.047480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":244,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079607+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:48:54.079607+0000","last_peered":"2023-10-09T04:48:54.079607+0000","last_clean":"2023-10-09T04:48:54.079607+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:48:54.079607+0000","last_undegraded":"2023-10-09T04:48:54.079607+0000","last_fullsized":"2023-10-09T04:48:54.079607+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"255'5632","reported_seq":5645,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:35.942067+0000","last_change":"2023-10-09T04:48:02.043240+0000","last_active":"2023-10-09T04:49:35.942067+0000","last_peered":"2023-10-09T04:49:35.942067+0000","last_clean":"2023-10-09T04:49:35.942067+0000","last_became_active":"2023-10-09T04:48:02.042897+0000","last_became_peered":"2023-10-09T04:48:02.042897+0000","last_unstale":"2023-10-09T04:49:35.942067+0000","last_undegraded":"2023-10-09T04:49:35.942067+0000","last_fullsized":"2023-10-09T04:49:35.942067+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369098752,"num_objects":5632,"num_object_clones":0,"num_object_copies":16896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5632,"num_write_kb":360448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":472,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.084773+0000","last_change":"2023-10-09T04:40:48.818879+0000","last_active":"2023-10-09T04:48:54.084773+0000","last_peered":"2023-10-09T04:48:54.084773+0000","last_clean":"2023-10-09T04:48:54.084773+0000","last_became_active":"2023-10-09T04:40:48.818073+0000","last_became_peered":"2023-10-09T04:40:48.818073+0000","last_unstale":"2023-10-09T04:48:54.084773+0000","last_undegraded":"2023-10-09T04:48:54.084773+0000","last_fullsized":"2023-10-09T04:48:54.084773+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.091235+0000","last_change":"2023-10-09T04:44:51.897117+0000","last_active":"2023-10-09T04:48:54.091235+0000","last_peered":"2023-10-09T04:48:54.091235+0000","last_clean":"2023-10-09T04:48:54.091235+0000","last_became_active":"2023-10-09T04:44:51.896672+0000","last_became_peered":"2023-10-09T04:44:51.896672+0000","last_unstale":"2023-10-09T04:48:54.091235+0000","last_undegraded":"2023-10-09T04:48:54.091235+0000","last_fullsized":"2023-10-09T04:48:54.091235+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:09.426260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":233,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.061955+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:49:19.061955+0000","last_peered":"2023-10-09T04:49:19.061955+0000","last_clean":"2023-10-09T04:49:19.061955+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:49:19.061955+0000","last_undegraded":"2023-10-09T04:49:19.061955+0000","last_fullsized":"2023-10-09T04:49:19.061955+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"255'5646","reported_seq":5659,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:35.943656+0000","last_change":"2023-10-09T04:48:02.046557+0000","last_active":"2023-10-09T04:49:35.943656+0000","last_peered":"2023-10-09T04:49:35.943656+0000","last_clean":"2023-10-09T04:49:35.943656+0000","last_became_active":"2023-10-09T04:48:02.046207+0000","last_became_peered":"2023-10-09T04:48:02.046207+0000","last_unstale":"2023-10-09T04:49:35.943656+0000","last_undegraded":"2023-10-09T04:49:35.943656+0000","last_fullsized":"2023-10-09T04:49:35.943656+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370016256,"num_objects":5646,"num_object_clones":0,"num_object_copies":16938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5646,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.086562+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:48:54.086562+0000","last_peered":"2023-10-09T04:48:54.086562+0000","last_clean":"2023-10-09T04:48:54.086562+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:48:54.086562+0000","last_undegraded":"2023-10-09T04:48:54.086562+0000","last_fullsized":"2023-10-09T04:48:54.086562+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":501,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.090697+0000","last_change":"2023-10-09T04:34:09.902165+0000","last_active":"2023-10-09T04:48:54.090697+0000","last_peered":"2023-10-09T04:48:54.090697+0000","last_clean":"2023-10-09T04:48:54.090697+0000","last_became_active":"2023-10-09T04:34:09.901439+0000","last_became_peered":"2023-10-09T04:34:09.901439+0000","last_unstale":"2023-10-09T04:48:54.090697+0000","last_undegraded":"2023-10-09T04:48:54.090697+0000","last_fullsized":"2023-10-09T04:48:54.090697+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:42.319543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.083942+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:48:54.083942+0000","last_peered":"2023-10-09T04:48:54.083942+0000","last_clean":"2023-10-09T04:48:54.083942+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:48:54.083942+0000","last_undegraded":"2023-10-09T04:48:54.083942+0000","last_fullsized":"2023-10-09T04:48:54.083942+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"255'5912","reported_seq":5925,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:40.134732+0000","last_change":"2023-10-09T04:48:02.156388+0000","last_active":"2023-10-09T04:49:40.134732+0000","last_peered":"2023-10-09T04:49:40.134732+0000","last_clean":"2023-10-09T04:49:40.134732+0000","last_became_active":"2023-10-09T04:48:02.155602+0000","last_became_peered":"2023-10-09T04:48:02.155602+0000","last_unstale":"2023-10-09T04:49:40.134732+0000","last_undegraded":"2023-10-09T04:49:40.134732+0000","last_fullsized":"2023-10-09T04:49:40.134732+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":250,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:48:01.007459+0000","last_clean_scrub_stamp":"2023-10-09T04:48:01.007459+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387448832,"num_objects":5912,"num_object_clones":0,"num_object_copies":17736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5912,"num_write_kb":378368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":335,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093683+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:48:54.093683+0000","last_peered":"2023-10-09T04:48:54.093683+0000","last_clean":"2023-10-09T04:48:54.093683+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:48:54.093683+0000","last_undegraded":"2023-10-09T04:48:54.093683+0000","last_fullsized":"2023-10-09T04:48:54.093683+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":218,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079691+0000","last_change":"2023-10-09T04:22:50.699187+0000","last_active":"2023-10-09T04:48:54.079691+0000","last_peered":"2023-10-09T04:48:54.079691+0000","last_clean":"2023-10-09T04:48:54.079691+0000","last_became_active":"2023-10-09T04:22:50.698763+0000","last_became_peered":"2023-10-09T04:22:50.698763+0000","last_unstale":"2023-10-09T04:48:54.079691+0000","last_undegraded":"2023-10-09T04:48:54.079691+0000","last_fullsized":"2023-10-09T04:48:54.079691+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:39.914095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":37,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079230+0000","last_change":"2023-10-09T04:43:53.980738+0000","last_active":"2023-10-09T04:48:54.079230+0000","last_peered":"2023-10-09T04:48:54.079230+0000","last_clean":"2023-10-09T04:48:54.079230+0000","last_became_active":"2023-10-09T04:43:53.980299+0000","last_became_peered":"2023-10-09T04:43:53.980299+0000","last_unstale":"2023-10-09T04:48:54.079230+0000","last_undegraded":"2023-10-09T04:48:54.079230+0000","last_fullsized":"2023-10-09T04:48:54.079230+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":219,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.080979+0000","last_change":"2023-10-09T04:40:47.823824+0000","last_active":"2023-10-09T04:48:54.080979+0000","last_peered":"2023-10-09T04:48:54.080979+0000","last_clean":"2023-10-09T04:48:54.080979+0000","last_became_active":"2023-10-09T04:40:47.817680+0000","last_became_peered":"2023-10-09T04:40:47.817680+0000","last_unstale":"2023-10-09T04:48:54.080979+0000","last_undegraded":"2023-10-09T04:48:54.080979+0000","last_fullsized":"2023-10-09T04:48:54.080979+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:07.850102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.097515+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:48:54.097515+0000","last_peered":"2023-10-09T04:48:54.097515+0000","last_clean":"2023-10-09T04:48:54.097515+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:48:54.097515+0000","last_undegraded":"2023-10-09T04:48:54.097515+0000","last_fullsized":"2023-10-09T04:48:54.097515+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":180,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.087889+0000","last_change":"2023-10-09T04:34:36.641944+0000","last_active":"2023-10-09T04:48:54.087889+0000","last_peered":"2023-10-09T04:48:54.087889+0000","last_clean":"2023-10-09T04:48:54.087889+0000","last_became_active":"2023-10-09T04:34:36.641610+0000","last_became_peered":"2023-10-09T04:34:36.641610+0000","last_unstale":"2023-10-09T04:48:54.087889+0000","last_undegraded":"2023-10-09T04:48:54.087889+0000","last_fullsized":"2023-10-09T04:48:54.087889+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.080163+0000","last_change":"2023-10-09T04:43:53.974915+0000","last_active":"2023-10-09T04:48:54.080163+0000","last_peered":"2023-10-09T04:48:54.080163+0000","last_clean":"2023-10-09T04:48:54.080163+0000","last_became_active":"2023-10-09T04:43:53.974611+0000","last_became_peered":"2023-10-09T04:43:53.974611+0000","last_unstale":"2023-10-09T04:48:54.080163+0000","last_undegraded":"2023-10-09T04:48:54.080163+0000","last_fullsized":"2023-10-09T04:48:54.080163+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":173,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.088309+0000","last_change":"2023-10-09T04:34:36.645066+0000","last_active":"2023-10-09T04:48:54.088309+0000","last_peered":"2023-10-09T04:48:54.088309+0000","last_clean":"2023-10-09T04:48:54.088309+0000","last_became_active":"2023-10-09T04:34:36.644626+0000","last_became_peered":"2023-10-09T04:34:36.644626+0000","last_unstale":"2023-10-09T04:48:54.088309+0000","last_undegraded":"2023-10-09T04:48:54.088309+0000","last_fullsized":"2023-10-09T04:48:54.088309+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.079232+0000","last_change":"2023-10-09T04:43:53.972398+0000","last_active":"2023-10-09T04:48:54.079232+0000","last_peered":"2023-10-09T04:48:54.079232+0000","last_clean":"2023-10-09T04:48:54.079232+0000","last_became_active":"2023-10-09T04:43:53.970226+0000","last_became_peered":"2023-10-09T04:43:53.970226+0000","last_unstale":"2023-10-09T04:48:54.079232+0000","last_undegraded":"2023-10-09T04:48:54.079232+0000","last_fullsized":"2023-10-09T04:48:54.079232+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":156,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.093601+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:48:54.093601+0000","last_peered":"2023-10-09T04:48:54.093601+0000","last_clean":"2023-10-09T04:48:54.093601+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:48:54.093601+0000","last_undegraded":"2023-10-09T04:48:54.093601+0000","last_fullsized":"2023-10-09T04:48:54.093601+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":218,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.088989+0000","last_change":"2023-10-09T04:44:51.889935+0000","last_active":"2023-10-09T04:49:19.088989+0000","last_peered":"2023-10-09T04:49:19.088989+0000","last_clean":"2023-10-09T04:49:19.088989+0000","last_became_active":"2023-10-09T04:44:51.889421+0000","last_became_peered":"2023-10-09T04:44:51.889421+0000","last_unstale":"2023-10-09T04:49:19.088989+0000","last_undegraded":"2023-10-09T04:49:19.088989+0000","last_fullsized":"2023-10-09T04:49:19.088989+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:53.165469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":249,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:49:19.061771+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:49:19.061771+0000","last_peered":"2023-10-09T04:49:19.061771+0000","last_clean":"2023-10-09T04:49:19.061771+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:49:19.061771+0000","last_undegraded":"2023-10-09T04:49:19.061771+0000","last_fullsized":"2023-10-09T04:49:19.061771+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":234,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-10-09T04:48:54.078954+0000","last_change":"2023-10-09T04:34:09.911306+0000","last_active":"2023-10-09T04:48:54.078954+0000","last_peered":"2023-10-09T04:48:54.078954+0000","last_clean":"2023-10-09T04:48:54.078954+0000","last_became_active":"2023-10-09T04:34:09.910941+0000","last_became_peered":"2023-10-09T04:34:09.910941+0000","last_unstale":"2023-10-09T04:48:54.078954+0000","last_undegraded":"2023-10-09T04:48:54.078954+0000","last_fullsized":"2023-10-09T04:48:54.078954+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6181945344,"num_objects":94329,"num_object_clones":0,"num_object_copies":282987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94329,"num_write_kb":6037056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18576244736,"data_stored":18576244736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94329,"ondisk_log_size":94329,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430702,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3881344,"kb_used_data":3291640,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89863808,"statfs":{"total":95995035648,"available":92020539392,"internally_reserved":0,"allocated":3370639360,"data_stored":3370055836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":2.4380000000000002,"15min":2.4279999999999999},"min":{"1min":0.53200000000000003,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":7.234,"5min":49.149999999999999,"15min":104.16200000000001},"last":3.367},{"interface":"front","average":{"1min":0.79300000000000004,"5min":2.1749999999999998,"15min":2.403},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":2.742,"5min":48.920999999999999,"15min":104.129},"last":0.63100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":2.5499999999999998,"15min":1.7210000000000001},"min":{"1min":0.61199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.32,"5min":58.816000000000003,"15min":58.816000000000003},"last":3.4980000000000002},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.5310000000000001,"15min":1.774},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.7430000000000003,"5min":58.840000000000003,"15min":58.840000000000003},"last":3.169}]},{"osd":2,"last update":"Mon Oct 9 04:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.4510000000000001,"15min":1.698},"min":{"1min":0.33900000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.496,"5min":6.5499999999999998,"15min":6.5499999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.875,"5min":1.7789999999999999,"15min":2.4119999999999999},"min":{"1min":0.42699999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.869,"5min":11.583,"15min":11.583},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":7.0510000000000002,"5min":2.8759999999999999,"15min":2.101},"min":{"1min":0.59499999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":102.02800000000001,"5min":102.02800000000001,"15min":102.02800000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":7.133,"5min":2.887,"15min":1.948},"min":{"1min":0.61199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":101.973,"5min":101.973,"15min":101.973},"last":0.66000000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.5109999999999999,"15min":1.278},"min":{"1min":0.47899999999999998,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":3.9510000000000001,"5min":11.933999999999999,"15min":11.933999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.496,"15min":1.3320000000000001},"min":{"1min":0.433,"5min":0.308,"15min":0.308},"max":{"1min":2.4039999999999999,"5min":8.452,"15min":10.696999999999999},"last":0.46700000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.6850000000000001,"15min":1.4550000000000001},"min":{"1min":0.45600000000000002,"5min":0.41899999999999998,"15min":0.35699999999999998},"max":{"1min":4.5979999999999999,"5min":11.268000000000001,"15min":11.268000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.452,"5min":1.8680000000000001,"15min":1.595},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.8159999999999998,"5min":15.138999999999999,"15min":15.138999999999999},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 04:49:33 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.466,"15min":1.427},"min":{"1min":0.377,"5min":0.377,"15min":0.36899999999999999},"max":{"1min":4.8040000000000003,"5min":11.986000000000001,"15min":11.986000000000001},"last":2.468},{"interface":"front","average":{"1min":1.605,"5min":1.5109999999999999,"15min":1.3660000000000001},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.35599999999999998},"max":{"1min":5.0910000000000002,"5min":12.013999999999999,"15min":12.013999999999999},"last":0.53100000000000003}]}]},{"osd":6,"up_from":146,"seq":627065225403,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1464376,"kb_used_data":1130224,"kb_used_omap":5,"kb_used_meta":334138,"kb_avail":92280776,"statfs":{"total":95995035648,"available":94495514624,"internally_reserved":0,"allocated":1157349376,"data_stored":1156773917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":342157920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.6549999999999998,"15min":3.5150000000000001},"min":{"1min":0.46300000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.786,"5min":76.189999999999998,"15min":214.82300000000001},"last":5.9530000000000003},{"interface":"front","average":{"1min":0.72399999999999998,"5min":2.823,"15min":3.5209999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":1.0169999999999999,"5min":76.165000000000006,"15min":213.34700000000001},"last":0.51300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":4.1459999999999999,"5min":2.7200000000000002,"15min":2.6909999999999998},"min":{"1min":0.39500000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":46.951000000000001,"5min":46.951000000000001,"15min":76.081000000000003},"last":2.419},{"interface":"front","average":{"1min":3.7429999999999999,"5min":2.113,"15min":2.6110000000000002},"min":{"1min":0.41599999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":45.884999999999998,"5min":45.884999999999998,"15min":75.546000000000006},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.6739999999999999,"15min":1.9550000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.0789999999999997,"5min":6.0570000000000004,"15min":6.0570000000000004},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.3500000000000001,"15min":1.498},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.4830000000000001,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.54900000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.722,"15min":1.726},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.9140000000000001,"5min":6.7599999999999998,"15min":11.673},"last":2.3860000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.544,"15min":1.724},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.522,"5min":9.3190000000000008,"15min":15.824},"last":0.432}]},{"osd":4,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.673,"15min":1.3009999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.248},"max":{"1min":1.696,"5min":13.699999999999999,"15min":13.699999999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.647,"15min":1.431},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.9820000000000002,"5min":7.0570000000000004,"15min":10.324},"last":2.335}]},{"osd":5,"last update":"Mon Oct 9 04:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.579,"15min":1.476},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.2730000000000001,"5min":9.3000000000000007,"15min":10.193},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.3,"5min":1.5980000000000001,"15min":1.627},"min":{"1min":0.53500000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.6879999999999997,"5min":8.4239999999999995,"15min":11.727},"last":4.6070000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:48:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.881,"15min":1.6040000000000001},"min":{"1min":0.42799999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.7650000000000001,"5min":16.527999999999999,"15min":16.527999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.012,"5min":1.905,"15min":1.526},"min":{"1min":0.61199999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.742,"5min":15.366,"15min":15.366},"last":0.98099999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958852,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4119828,"kb_used_data":3431308,"kb_used_omap":6,"kb_used_meta":688505,"kb_avail":89625324,"statfs":{"total":95995035648,"available":91776331776,"internally_reserved":0,"allocated":3513659392,"data_stored":3512650669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":705029132},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.0139999999999998,"15min":2.7930000000000001},"min":{"1min":0.22800000000000001,"5min":0.19700000000000001,"15min":0.17499999999999999},"max":{"1min":4.2300000000000004,"5min":37.462000000000003,"15min":91.025999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.9339999999999999,"15min":2.8530000000000002},"min":{"1min":0.28599999999999998,"5min":0.23899999999999999,"15min":0.17999999999999999},"max":{"1min":1.9730000000000001,"5min":43.777999999999999,"15min":91.536000000000001},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 04:48:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":2.097,"15min":2.5030000000000001},"min":{"1min":0.27800000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.5580000000000001,"5min":15.336,"15min":15.336},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.7829999999999999,"15min":2.2919999999999998},"min":{"1min":0.26300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":3.3069999999999999,"5min":13.444000000000001,"15min":13.444000000000001},"last":0.54500000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:48:52 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.667,"15min":1.7769999999999999},"min":{"1min":0.35599999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.9039999999999999,"5min":10.401999999999999,"15min":14.369},"last":2.823},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.764,"15min":1.8200000000000001},"min":{"1min":0.376,"5min":0.29699999999999999,"15min":0.20200000000000001},"max":{"1min":7.1120000000000001,"5min":19.617999999999999,"15min":19.617999999999999},"last":2.8769999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.55,"15min":1.4590000000000001},"min":{"1min":0.35699999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":5.7699999999999996,"5min":13.362,"15min":13.362},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.502,"5min":1.6839999999999999,"15min":1.514},"min":{"1min":0.51400000000000001,"5min":0.316,"15min":0.316},"max":{"1min":3.6400000000000001,"5min":8.6890000000000001,"15min":12.209},"last":0.64700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.6839999999999999,"15min":1.6850000000000001},"min":{"1min":0.35799999999999998,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":4.8339999999999996,"5min":14.494999999999999,"15min":14.494999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.7110000000000001,"15min":1.593},"min":{"1min":0.51500000000000001,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":2.0510000000000002,"5min":13.413,"15min":13.413},"last":0.78400000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.6990000000000001,"15min":1.728},"min":{"1min":0.45500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.883,"5min":14.545,"15min":14.545},"last":3.883},{"interface":"front","average":{"1min":1.375,"5min":1.806,"15min":1.8080000000000001},"min":{"1min":0.42999999999999999,"5min":0.40500000000000003,"15min":0.39900000000000002},"max":{"1min":3.931,"5min":14.577,"15min":14.577},"last":3.931}]},{"osd":7,"last update":"Mon Oct 9 04:49:38 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.978,"15min":2.093},"min":{"1min":0.45700000000000002,"5min":0.374,"15min":0.34399999999999997},"max":{"1min":7.5819999999999999,"5min":13.474,"15min":144.613},"last":1.6899999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.8029999999999999,"15min":2.0449999999999999},"min":{"1min":0.57999999999999996,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.8959999999999999,"5min":8.6500000000000004,"15min":145.53700000000001},"last":0.67800000000000005}]}]},{"osd":0,"up_from":88,"seq":377957122377,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2174824,"kb_used_data":1517152,"kb_used_omap":9,"kb_used_meta":657654,"kb_avail":91570328,"statfs":{"total":95995035648,"available":93768015872,"internally_reserved":0,"allocated":1553563648,"data_stored":1552540386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":673438193},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.456,"15min":2.1080000000000001},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":1.383,"5min":7.5270000000000001,"15min":223.98500000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.282,"5min":1.714,"15min":2.294},"min":{"1min":0.317,"5min":0.22700000000000001,"15min":0.19900000000000001},"max":{"1min":4.6059999999999999,"5min":10.500999999999999,"15min":223.13},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":2.04,"15min":2.3260000000000001},"min":{"1min":0.29599999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.5649999999999999,"5min":17.109000000000002,"15min":17.109000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.097,"5min":2.2730000000000001,"15min":2.677},"min":{"1min":0.36699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4020000000000001,"5min":12.57,"15min":12.57},"last":0.68799999999999994}]},{"osd":3,"last update":"Mon Oct 9 04:48:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.492,"15min":1.8100000000000001},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.23400000000000001},"max":{"1min":1.2849999999999999,"5min":10.933999999999999,"15min":64.622},"last":2.5209999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.841,"15min":2.0619999999999998},"min":{"1min":0.45900000000000002,"5min":0.23999999999999999,"15min":0.218},"max":{"1min":2.661,"5min":10.398999999999999,"15min":64.540999999999997},"last":2.3700000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:49:15 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":1.9370000000000001,"15min":1.7390000000000001},"min":{"1min":0.377,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.5390000000000001,"5min":16.84,"15min":61.549999999999997},"last":1.409},{"interface":"front","average":{"1min":1.048,"5min":1.734,"15min":1.5620000000000001},"min":{"1min":0.371,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":3.657,"5min":39.957999999999998,"15min":61.902999999999999},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:48:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.7390000000000001,"15min":1.9570000000000001},"min":{"1min":0.55400000000000005,"5min":0.35399999999999998,"15min":0.311},"max":{"1min":5.8040000000000003,"5min":7.1500000000000004,"15min":61.972999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.5720000000000001,"15min":1.895},"min":{"1min":0.51900000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0339999999999998,"5min":10.215999999999999,"15min":62.417000000000002},"last":1.9319999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:49:35 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":1.4259999999999999,"15min":1.758},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.34699999999999998},"max":{"1min":5.0890000000000004,"5min":7.5759999999999996,"15min":62.462000000000003},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.899,"15min":2.0289999999999999},"min":{"1min":0.502,"5min":0.39400000000000002,"15min":0.32100000000000001},"max":{"1min":7.9210000000000003,"5min":9.1969999999999992,"15min":61.808},"last":0.79300000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:49:15 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.9850000000000001,"15min":1.988},"min":{"1min":0.62,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":7.851,"5min":7.851,"15min":62.006},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.363,"5min":1.8899999999999999,"15min":1.899},"min":{"1min":0.39900000000000002,"5min":0.372,"15min":0.32300000000000001},"max":{"1min":3.5870000000000002,"5min":10.804,"15min":61.606999999999999},"last":1.5589999999999999}]}]},{"osd":2,"up_from":234,"seq":1005022347323,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1363480,"kb_used_data":1144464,"kb_used_omap":13,"kb_used_meta":218994,"kb_avail":92381672,"statfs":{"total":95995035648,"available":94598832128,"internally_reserved":0,"allocated":1171931136,"data_stored":1170919425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":224250751},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.0910000000000002,"15min":2.8010000000000002},"min":{"1min":0.25700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.5859999999999999,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":1.895},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.992,"15min":2.7269999999999999},"min":{"1min":0.251,"5min":0.249,"15min":0.249},"max":{"1min":6.827,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.93899999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":2.0219999999999998,"15min":2.4319999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.103,"5min":7.907,"15min":7.907},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.605,"15min":1.8129999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.327,"5min":7.8780000000000001,"15min":7.8780000000000001},"last":0.53900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.536,"15min":1.623},"min":{"1min":0.41199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.141,"5min":14.340999999999999,"15min":14.340999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.76400000000000001,"5min":2.0649999999999999,"15min":2.605},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":2.544,"5min":9.6039999999999992,"15min":9.6039999999999992},"last":2.218}]},{"osd":4,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.7609999999999999,"15min":2.0459999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.976,"5min":6.6520000000000001,"15min":6.6520000000000001},"last":2.0880000000000001},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.7410000000000001,"15min":1.962},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.55,"5min":7.9390000000000001,"15min":7.9390000000000001},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.677,"15min":2.0710000000000002},"min":{"1min":0.39500000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.54,"5min":9.1899999999999995,"15min":9.1899999999999995},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.2360000000000002,"15min":2.766},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":2.4740000000000002,"5min":12.295999999999999,"15min":12.295999999999999},"last":2.1440000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.829,"15min":2.1720000000000002},"min":{"1min":0.44800000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.2709999999999999,"5min":8.2409999999999997,"15min":8.2409999999999997},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.573,"15min":1.657},"min":{"1min":0.46500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.3210000000000002,"5min":7.3040000000000003,"15min":7.3040000000000003},"last":0.82399999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":2.0880000000000001,"15min":2.2210000000000001},"min":{"1min":0.54200000000000004,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.913,"5min":14.161,"15min":14.161},"last":2.1789999999999998},{"interface":"front","average":{"1min":0.86399999999999999,"5min":1.851,"15min":1.976},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3520000000000001,"5min":10.996,"15min":10.996},"last":0.78600000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631798,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3740956,"kb_used_data":3051348,"kb_used_omap":5,"kb_used_meta":689594,"kb_avail":90004196,"statfs":{"total":95995035648,"available":92164296704,"internally_reserved":0,"allocated":3124580352,"data_stored":3123565601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":706144430},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.4770000000000001,"15min":2.8010000000000002},"min":{"1min":0.24299999999999999,"5min":0.19500000000000001,"15min":0.154},"max":{"1min":1.252,"5min":8.5220000000000002,"15min":341.28699999999998},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.71199999999999997,"5min":1.474,"15min":2.7450000000000001},"min":{"1min":0.28999999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.3169999999999999,"5min":9.5079999999999991,"15min":341.57799999999997},"last":2.8599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.464,"15min":1.653},"min":{"1min":0.26500000000000001,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":4.1609999999999996,"5min":17.172999999999998,"15min":47.637999999999998},"last":2.7570000000000001},{"interface":"front","average":{"1min":1.159,"5min":2.0099999999999998,"15min":1.8759999999999999},"min":{"1min":0.23200000000000001,"5min":0.21299999999999999,"15min":0.188},"max":{"1min":5.8529999999999998,"5min":17.125,"15min":47.616999999999997},"last":0.76000000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:48:59 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.4370000000000001,"15min":1.611},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.8180000000000001,"5min":6.4240000000000004,"15min":6.4240000000000004},"last":2.6709999999999998},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.5840000000000001,"15min":1.8160000000000001},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.3929999999999998,"5min":10.154999999999999,"15min":10.154999999999999},"last":2.903}]},{"osd":4,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.506,"15min":1.375},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":1.2809999999999999,"5min":8.4350000000000005,"15min":12.981},"last":3.0739999999999998},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.716,"15min":1.452},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.28000000000000003},"max":{"1min":1.335,"5min":9.4359999999999999,"15min":9.4359999999999999},"last":3.7120000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:49:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.8080000000000001,"15min":1.681},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32300000000000001},"max":{"1min":3.0529999999999999,"5min":7.9649999999999999,"15min":8.0109999999999992},"last":3.855},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.637,"15min":1.623},"min":{"1min":0.47199999999999998,"5min":0.35699999999999998,"15min":0.32700000000000001},"max":{"1min":2.3599999999999999,"5min":9.4800000000000004,"15min":9.4800000000000004},"last":3.6749999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:48:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.2669999999999999,"15min":1.669},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32100000000000001},"max":{"1min":1.5189999999999999,"5min":9.0020000000000007,"15min":11.842000000000001},"last":2.7160000000000002},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.4630000000000001,"15min":1.5649999999999999},"min":{"1min":0.497,"5min":0.38700000000000001,"15min":0.35399999999999998},"max":{"1min":1.478,"5min":8.4860000000000007,"15min":13.004},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.6639999999999999,"15min":2.524},"min":{"1min":0.65500000000000003,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":3.718,"5min":6.7729999999999997,"15min":198.012},"last":3.2559999999999998},{"interface":"front","average":{"1min":1.325,"5min":1.6970000000000001,"15min":2.4529999999999998},"min":{"1min":0.46700000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.0449999999999999,"5min":8.4469999999999992,"15min":198.05199999999999},"last":3.2320000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019400,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722232,"kb_used_data":1132528,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92022920,"statfs":{"total":95995035648,"available":94231470080,"internally_reserved":0,"allocated":1159708672,"data_stored":1159132726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":2.4159999999999999,"15min":1.7250000000000001},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26100000000000001},"max":{"1min":3.6899999999999999,"5min":52.106999999999999,"15min":52.106999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.3340000000000001,"15min":1.9610000000000001},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":1.6399999999999999,"5min":52.289999999999999,"15min":52.289999999999999},"last":1.655}]},{"osd":1,"last update":"Mon Oct 9 04:49:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0069999999999997,"5min":2.3620000000000001,"15min":1.714},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":52.524999999999999,"5min":52.524999999999999,"15min":52.524999999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":4.2480000000000002,"5min":2.702,"15min":1.8089999999999999},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":51.920000000000002,"5min":51.920000000000002,"15min":51.920000000000002},"last":0.433}]},{"osd":2,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.645,"15min":1.919},"min":{"1min":0.372,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.734,"5min":10.249000000000001,"15min":10.249000000000001},"last":0.755},{"interface":"front","average":{"1min":0.749,"5min":2.2229999999999999,"15min":2.3519999999999999},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7549999999999999,"5min":10.535,"15min":10.535},"last":0.46100000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.296,"15min":1.4530000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.315},"max":{"1min":3.3439999999999999,"5min":10.42,"15min":10.42},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.016,"5min":1.425,"15min":1.4259999999999999},"min":{"1min":0.41599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3690000000000002,"5min":8.4909999999999997,"15min":9.484},"last":0.79400000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":5.21,"5min":2.6699999999999999,"15min":1.6599999999999999},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.251},"max":{"1min":78.885000000000005,"5min":78.885000000000005,"15min":78.885000000000005},"last":2.8439999999999999},{"interface":"front","average":{"1min":5.359,"5min":2.3820000000000001,"15min":1.641},"min":{"1min":0.41699999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":78.472999999999999,"5min":78.472999999999999,"15min":78.472999999999999},"last":2.6259999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":1.1140000000000001,"15min":1.24},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.77,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.375},{"interface":"front","average":{"1min":1.101,"5min":1.3380000000000001,"15min":1.2569999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":3.8530000000000002,"5min":8.5470000000000006,"15min":10.257999999999999},"last":3.8530000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:49:36 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.5569999999999999,"15min":1.319},"min":{"1min":0.438,"5min":0.39800000000000002,"15min":0.33600000000000002},"max":{"1min":6.2149999999999999,"5min":7.4029999999999996,"15min":9.0370000000000008},"last":2.5350000000000001},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.4910000000000001,"15min":1.417},"min":{"1min":0.45600000000000002,"5min":0.377,"15min":0.35099999999999998},"max":{"1min":3.6520000000000001,"5min":8.9670000000000005,"15min":13.349},"last":2.5510000000000002}]}]},{"osd":5,"up_from":150,"seq":644245094582,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4055720,"kb_used_data":3413536,"kb_used_omap":6,"kb_used_meta":642169,"kb_avail":89689432,"statfs":{"total":95995035648,"available":91841978368,"internally_reserved":0,"allocated":3495460864,"data_stored":3494896750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":657581331},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":2.1699999999999999,"15min":1.7729999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":40.564,"5min":40.564,"15min":40.564},"last":46.085000000000001},{"interface":"front","average":{"1min":3.5310000000000001,"5min":2.2879999999999998,"15min":1.98},"min":{"1min":0.38200000000000001,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":40.701000000000001,"5min":40.701000000000001,"15min":40.701000000000001},"last":46.127000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":3.5819999999999999,"5min":1.978,"15min":1.7589999999999999},"min":{"1min":0.45300000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":45.043999999999997,"5min":45.043999999999997,"15min":45.043999999999997},"last":47.026000000000003},{"interface":"front","average":{"1min":3.75,"5min":2.165,"15min":1.9139999999999999},"min":{"1min":0.60899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":40.442999999999998,"5min":40.442999999999998,"15min":40.442999999999998},"last":47.558999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:48:59 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.5529999999999999,"15min":1.651},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.3199999999999998,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.8200000000000001,"15min":2.238},"min":{"1min":0.38300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.3010000000000002,"5min":13.536,"15min":13.536},"last":0.81499999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":5.5540000000000003,"5min":2.2709999999999999,"15min":1.8120000000000001},"min":{"1min":0.42899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":81.308999999999997,"5min":81.308999999999997,"15min":81.308999999999997},"last":0.89300000000000002},{"interface":"front","average":{"1min":5.8529999999999998,"5min":2.6190000000000002,"15min":2.141},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.36299999999999999},"max":{"1min":81.091999999999999,"5min":81.091999999999999,"15min":81.091999999999999},"last":0.57399999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.629,"15min":1.5069999999999999},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.3109999999999999,"5min":7.0759999999999996,"15min":11.359},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.075,"5min":1.577,"15min":1.3220000000000001},"min":{"1min":0.46600000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.4089999999999998,"5min":10.709,"15min":10.709},"last":0.76200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.518,"15min":1.4530000000000001},"min":{"1min":0.46800000000000003,"5min":0.35899999999999999,"15min":0.34399999999999997},"max":{"1min":2.2229999999999999,"5min":10.77,"15min":11.087},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.5009999999999999,"15min":1.514},"min":{"1min":0.42899999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.7999999999999998,"5min":6.9400000000000004,"15min":11.222},"last":1.1639999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:49:13 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":1.8049999999999999,"15min":1.8100000000000001},"min":{"1min":0.46800000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.5330000000000004,"5min":8.6820000000000004,"15min":39.771000000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.296,"5min":1.8959999999999999,"15min":1.853},"min":{"1min":0.45600000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.54,"5min":14.170999999999999,"15min":39.905000000000001},"last":0.92400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1561526272,"data_stored":1561526272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3511091200,"data_stored":3511091200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1175191552,"data_stored":1175191552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3126919168,"data_stored":3126919168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1163329536,"data_stored":1163329536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3508535296,"data_stored":3508535296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1157038080,"data_stored":1157038080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3372613632,"data_stored":3372613632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:49:44.493 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:49:44.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T04:49:44.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:44.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:44.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:44.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:44.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:44.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:44.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:44.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:44.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:44.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:45.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:45.103 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:45.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:45.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:45.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:45.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 99557 99541 79.7458 51.6875 0.0161154 0.012534 2023-10-09T04:49:45.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:45.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:45.249 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:49:45.284 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:49:45.323 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:49:45.342 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:45 smithi082 bash[77063]: audit 2023-10-09T04:49:44.291072+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:45.342 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:45 smithi082 bash[79733]: audit 2023-10-09T04:49:44.291072+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:45.343 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:45.366 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:49:45.398 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:49:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:45 smithi153 bash[59565]: audit 2023-10-09T04:49:44.291072+0000 mon.a (mon.0) 1548 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:45.444 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:49:45.479 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:49:45.519 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:49:45.521 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.522 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:49:45.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:45.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:45.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:45.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:45.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:45.845 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:45.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:45.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:45.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:46.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:46.085 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:49:46.126 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:46.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:46 smithi153 bash[59565]: cluster 2023-10-09T04:49:44.736531+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:49:46.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:46 smithi153 bash[59565]: audit 2023-10-09T04:49:45.193898+0000 mon.a (mon.0) 1549 : audit 0 from='client.? 172.21.15.82:0/2831428656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:46.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:46 smithi153 bash[59565]: audit 2023-10-09T04:49:45.957624+0000 mon.a (mon.0) 1550 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:46.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:46.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 100289 100273 79.3154 45.75 0.0149719 0.0125823 2023-10-09T04:49:46.266 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:46.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:46.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:46.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[77063]: cluster 2023-10-09T04:49:44.736531+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[77063]: audit 2023-10-09T04:49:45.193898+0000 mon.a (mon.0) 1549 : audit 0 from='client.? 172.21.15.82:0/2831428656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[77063]: audit 2023-10-09T04:49:45.957624+0000 mon.a (mon.0) 1550 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[79733]: cluster 2023-10-09T04:49:44.736531+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[79733]: audit 2023-10-09T04:49:45.193898+0000 mon.a (mon.0) 1549 : audit 0 from='client.? 172.21.15.82:0/2831428656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:46.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:46 smithi082 bash[79733]: audit 2023-10-09T04:49:45.957624+0000 mon.a (mon.0) 1550 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:46.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.569 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:46.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:46.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:46.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:46.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:46.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:46.815 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:49:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:46.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:46.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:46.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:47.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[79733]: cluster 2023-10-09T04:49:46.058181+0000 mon.a (mon.0) 1551 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:47.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[79733]: cluster 2023-10-09T04:49:46.058221+0000 mon.a (mon.0) 1552 : cluster 1 Cluster is now healthy 2023-10-09T04:49:47.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[79733]: cluster 2023-10-09T04:49:46.071345+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T04:49:47.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[79733]: audit 2023-10-09T04:49:46.075729+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:47.067 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[79733]: cluster 2023-10-09T04:49:46.075965+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:49:47.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:47.102 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:47.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:47.214 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:49:47.214173+0000 min lat: 0.00135091 max lat: 0.280251 avg lat: 0.0126549 2023-10-09T04:49:47.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:49:47.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 101140 101124 78.9888 53.1875 0.0123071 0.0126549 2023-10-09T04:49:47.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:47.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:47.364 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:47 smithi153 bash[59565]: cluster 2023-10-09T04:49:46.058181+0000 mon.a (mon.0) 1551 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:47.364 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:47 smithi153 bash[59565]: cluster 2023-10-09T04:49:46.058221+0000 mon.a (mon.0) 1552 : cluster 1 Cluster is now healthy 2023-10-09T04:49:47.364 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:47 smithi153 bash[59565]: cluster 2023-10-09T04:49:46.071345+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T04:49:47.365 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:47 smithi153 bash[59565]: audit 2023-10-09T04:49:46.075729+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:47.365 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:47 smithi153 bash[59565]: cluster 2023-10-09T04:49:46.075965+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:49:47.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:47.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[77063]: cluster 2023-10-09T04:49:46.058181+0000 mon.a (mon.0) 1551 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[77063]: cluster 2023-10-09T04:49:46.058221+0000 mon.a (mon.0) 1552 : cluster 1 Cluster is now healthy 2023-10-09T04:49:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[77063]: cluster 2023-10-09T04:49:46.071345+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T04:49:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[77063]: audit 2023-10-09T04:49:46.075729+0000 mon.a (mon.0) 1553 : audit 1 from='client.? 172.21.15.82:0/201178874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:47.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:47 smithi082 bash[77063]: cluster 2023-10-09T04:49:46.075965+0000 mon.a (mon.0) 1554 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:49:47.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:47.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:47.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:47.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:47.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:47.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:48.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:48.065 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:48.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:48.179 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:48 smithi082 bash[77063]: cluster 2023-10-09T04:49:46.736952+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:49:48.180 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:48 smithi082 bash[79733]: cluster 2023-10-09T04:49:46.736952+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:49:48.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:48.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 102022 102006 78.6941 55.125 0.00985879 0.0127019 2023-10-09T04:49:48.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:48.313 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:48 smithi153 bash[59565]: cluster 2023-10-09T04:49:46.736952+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T04:49:48.336 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:48.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:48.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:48.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:48.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:48.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:48.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:48.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:48.838 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:48.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:48.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:48.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:49.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:49.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:49.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:49.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 102894 102878 78.399 54.5 0.0156022 0.0127498 2023-10-09T04:49:49.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:49.261 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:49.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:49.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:49.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:49.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:49.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.672 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:49.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:49.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:49.918 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:49] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T04:49:49.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:49.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:50.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:50.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:50.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:49:50.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:49:50.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:50.182 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:50 smithi082 bash[79733]: cluster 2023-10-09T04:49:48.737455+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:50.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:50 smithi082 bash[77063]: cluster 2023-10-09T04:49:48.737455+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:50.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:50.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 16 103705 103689 78.0651 50.6875 0.0265978 0.0128029 2023-10-09T04:49:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:50.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:50.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:50 smithi153 bash[59565]: cluster 2023-10-09T04:49:48.737455+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-09T04:49:50.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:50.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:50.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:50.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:50.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:50.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:50.965 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:50.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:51.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.103 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:51.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:51.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 104526 104510 77.7466 51.3125 0.0106452 0.0128565 2023-10-09T04:49:51.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:51.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:51.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:51.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:51.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:51.656 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:51.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:51.782 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:51.887 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:51.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:51.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:52.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:52.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:52.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:52.189 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:52.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 105376 105360 77.4569 53.125 0.0202964 0.0129049 2023-10-09T04:49:52.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:52.319 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:52.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:52 smithi153 bash[59565]: cluster 2023-10-09T04:49:50.738061+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T04:49:52.427 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:52.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:52 smithi082 bash[79733]: cluster 2023-10-09T04:49:50.738061+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T04:49:52.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:52 smithi082 bash[77063]: cluster 2023-10-09T04:49:50.738061+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T04:49:52.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:52.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:52.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:52.680 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:52.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:52.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:52.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:52.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:52.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:52.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:53.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:53.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:53.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:53.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 106223 106207 77.1717 52.9375 0.0188519 0.0129528 2023-10-09T04:49:53.258 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:53.292 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:53.400 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:53.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:53.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:53.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:53.673 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:53.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:53.817 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:53.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:53.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:53.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:54.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:54.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:54.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 107054 107038 76.8817 51.9375 0.0168051 0.0130017 2023-10-09T04:49:54.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:54 smithi082 bash[77063]: cluster 2023-10-09T04:49:52.738492+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-10-09T04:49:54.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:54 smithi082 bash[79733]: cluster 2023-10-09T04:49:52.738492+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-10-09T04:49:54.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:54.307 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:54.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:54.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.565 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:54 smithi153 bash[59565]: cluster 2023-10-09T04:49:52.738492+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s 2023-10-09T04:49:54.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:54.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:54.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:54.953 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:54.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:54.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:55.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:55.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 107822 107806 76.5534 48 0.0131203 0.0130577 2023-10-09T04:49:55.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:55.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:55.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:55.390 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:55.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:55.493 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:55.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:55.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:55.821 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:55.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:55.924 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.949 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:55.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:56.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:56.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:56.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:56.215 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 16 108566 108550 76.2158 46.5 0.0212894 0.0131152 2023-10-09T04:49:56.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:56.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:56.349 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:56 smithi153 bash[59565]: cluster 2023-10-09T04:49:54.740737+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T04:49:56.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:56.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:56.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:56.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:56 smithi082 bash[77063]: cluster 2023-10-09T04:49:54.740737+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T04:49:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:56 smithi082 bash[79733]: cluster 2023-10-09T04:49:54.740737+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1148: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T04:49:56.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:56.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:56.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:56.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:56.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:56.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:56.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:56.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:56.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:57.097 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:57.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 90.0126 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 109358 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 75.9324 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 29.6034 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 179.438 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 45.75 2023-10-09T04:49:57.216 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 1214 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 473.654 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 2871 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 732 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.0131651 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.0119594 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 0.280251 2023-10-09T04:49:57.217 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00135091 2023-10-09T04:49:57.223 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:49:57.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:57.251 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:57.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:57.355 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:57.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:57.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:57.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:57.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:57.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:57.728 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:57.833 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:57.870 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:57.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:57.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.002 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:58.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:58.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:58.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:58.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:49:58 smithi082 bash[77063]: cluster 2023-10-09T04:49:56.742263+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:58.246 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:49:58 smithi082 bash[79733]: cluster 2023-10-09T04:49:56.742263+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:58.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:58.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:58.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:49:58 smithi153 bash[59565]: cluster 2023-10-09T04:49:56.742263+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s 2023-10-09T04:49:58.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:58.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:58.630 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:58.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:58.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:58.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:58.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:58.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:59.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:59.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:59.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:59.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.323 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:59.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:59.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:59.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:59.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:59.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:59.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:59.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:59.843 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:49:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:49:59] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T04:49:59.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:59.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:49:59.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:00.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:00.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[77063]: cluster 2023-10-09T04:49:58.742759+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T04:50:00.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[77063]: audit 2023-10-09T04:49:59.295764+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:00.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[77063]: audit 2023-10-09T04:49:59.297381+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:00.105 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[77063]: cluster 2023-10-09T04:50:00.000105+0000 mon.a (mon.0) 1557 : cluster 1 overall HEALTH_OK 2023-10-09T04:50:00.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[79733]: cluster 2023-10-09T04:49:58.742759+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T04:50:00.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[79733]: audit 2023-10-09T04:49:59.295764+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:00.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[79733]: audit 2023-10-09T04:49:59.297381+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:00.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:00 smithi082 bash[79733]: cluster 2023-10-09T04:50:00.000105+0000 mon.a (mon.0) 1557 : cluster 1 overall HEALTH_OK 2023-10-09T04:50:00.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:00.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:00.406 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:00 smithi153 bash[59565]: cluster 2023-10-09T04:49:58.742759+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1150: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-09T04:50:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:00 smithi153 bash[59565]: audit 2023-10-09T04:49:59.295764+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:00 smithi153 bash[59565]: audit 2023-10-09T04:49:59.297381+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:00 smithi153 bash[59565]: cluster 2023-10-09T04:50:00.000105+0000 mon.a (mon.0) 1557 : cluster 1 overall HEALTH_OK 2023-10-09T04:50:00.438 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:00.544 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.573 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:00.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:00.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:00.813 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:00.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:00.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:00.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:00.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:01.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:01.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:01.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:01.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:01.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:01.489 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:01.621 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:01.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:01.747 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:01.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:01.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:02.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:02.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:02.148 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:02 smithi082 bash[77063]: cluster 2023-10-09T04:50:00.743467+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 942 op/s 2023-10-09T04:50:02.149 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:02 smithi082 bash[79733]: cluster 2023-10-09T04:50:00.743467+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 942 op/s 2023-10-09T04:50:02.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:02.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.311 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:02 smithi153 bash[59565]: cluster 2023-10-09T04:50:00.743467+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 942 op/s 2023-10-09T04:50:02.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:02.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:02.463 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:02.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:02.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:02.695 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:02.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:02.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:02.858 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:02.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:02.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:03.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:03.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:03.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:03.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:03.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:03.533 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:03.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:03.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:03.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:03.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:04.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:04.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:04.119 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:04 smithi082 bash[77063]: cluster 2023-10-09T04:50:02.743918+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 742 op/s 2023-10-09T04:50:04.119 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:04 smithi153 bash[59565]: cluster 2023-10-09T04:50:02.743918+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 742 op/s 2023-10-09T04:50:04.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:04 smithi082 bash[79733]: cluster 2023-10-09T04:50:02.743918+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 742 op/s 2023-10-09T04:50:04.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:04.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:04.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:04.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:04.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:04.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:04.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:04.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:04.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:04.805 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:04.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:04.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:04.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:05.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:05.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:05.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:05.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:05.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:05.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:05.476 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:05.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:05.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:05.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:05.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:05.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:05.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:05.883 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:05.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:05.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:05.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:06.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:06.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:06.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:06.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:06.171 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:06.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.309 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:06.388 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:06 smithi082 bash[77063]: cluster 2023-10-09T04:50:04.744515+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 881 op/s 2023-10-09T04:50:06.389 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:06 smithi082 bash[79733]: cluster 2023-10-09T04:50:04.744515+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 881 op/s 2023-10-09T04:50:06.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:06 smithi153 bash[59565]: cluster 2023-10-09T04:50:04.744515+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 881 op/s 2023-10-09T04:50:06.475 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:06.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.606 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:06.665 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:06.665 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:06.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:06.744 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:06.849 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.878 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:06.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:06.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:07.014 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:07.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.155 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 9 pg_num to 26 2023-10-09T04:50:07.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:07.264 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:07.295 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:07.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:07.407 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:07 smithi153 bash[59565]: audit 2023-10-09T04:50:06.729061+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.82:0/3439237505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.407 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:07 smithi153 bash[59565]: audit 2023-10-09T04:50:06.729782+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:07 smithi082 bash[77063]: audit 2023-10-09T04:50:06.729061+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.82:0/3439237505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:07 smithi082 bash[77063]: audit 2023-10-09T04:50:06.729782+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:07 smithi082 bash[79733]: audit 2023-10-09T04:50:06.729061+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.82:0/3439237505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:07 smithi082 bash[79733]: audit 2023-10-09T04:50:06.729782+0000 mon.a (mon.0) 1558 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:50:07.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:07.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:07.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.678 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:07.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:07.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:07.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:07.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:07.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:07.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:08.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:08.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:08.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: cluster 2023-10-09T04:50:06.745048+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 836 op/s 2023-10-09T04:50:08.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: cluster 2023-10-09T04:50:07.141260+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: cluster 2023-10-09T04:50:07.143639+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: audit 2023-10-09T04:50:07.146812+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:50:08.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: cluster 2023-10-09T04:50:07.147045+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:50:08.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:08 smithi153 bash[59565]: cluster 2023-10-09T04:50:07.180760+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:08.185 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: cluster 2023-10-09T04:50:06.745048+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 836 op/s 2023-10-09T04:50:08.185 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: cluster 2023-10-09T04:50:07.141260+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: cluster 2023-10-09T04:50:07.143639+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: audit 2023-10-09T04:50:07.146812+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: cluster 2023-10-09T04:50:07.147045+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[77063]: cluster 2023-10-09T04:50:07.180760+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: cluster 2023-10-09T04:50:06.745048+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 836 op/s 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: cluster 2023-10-09T04:50:07.141260+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: cluster 2023-10-09T04:50:07.143639+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: audit 2023-10-09T04:50:07.146812+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:50:08.186 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: cluster 2023-10-09T04:50:07.147045+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:50:08.187 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:08 smithi082 bash[79733]: cluster 2023-10-09T04:50:07.180760+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T04:50:08.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:08.308 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:08.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:08.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:08.575 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:08.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:08.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:08.831 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:08.966 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:08.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:08.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:09.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:09.120 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:09.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:09.223 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:09.261 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:09 smithi153 bash[59565]: cluster 2023-10-09T04:50:08.745549+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 826 op/s 2023-10-09T04:50:09.261 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:09 smithi153 bash[59565]: audit 2023-10-09T04:50:08.748489+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:50:09.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:09.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:09.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:09.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:09 smithi082 bash[77063]: cluster 2023-10-09T04:50:08.745549+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 826 op/s 2023-10-09T04:50:09.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:09 smithi082 bash[77063]: audit 2023-10-09T04:50:08.748489+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:50:09.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:09 smithi082 bash[79733]: cluster 2023-10-09T04:50:08.745549+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 826 op/s 2023-10-09T04:50:09.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:09 smithi082 bash[79733]: audit 2023-10-09T04:50:08.748489+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:50:09.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:09.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:09.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:09.732 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:09] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-10-09T04:50:09.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:09.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:09.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:10.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:10.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:10.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:10.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:10.333 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:10.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:10 smithi153 bash[59565]: cluster 2023-10-09T04:50:09.149986+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:10 smithi153 bash[59565]: audit 2023-10-09T04:50:09.157241+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:50:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:10 smithi153 bash[59565]: cluster 2023-10-09T04:50:09.157410+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:50:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:10 smithi153 bash[59565]: cluster 2023-10-09T04:50:09.182633+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:10 smithi153 bash[59565]: cluster 2023-10-09T04:50:09.184635+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:10.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:10.555 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[77063]: cluster 2023-10-09T04:50:09.149986+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.555 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[77063]: audit 2023-10-09T04:50:09.157241+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:50:10.555 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[77063]: cluster 2023-10-09T04:50:09.157410+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[77063]: cluster 2023-10-09T04:50:09.182633+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[77063]: cluster 2023-10-09T04:50:09.184635+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[79733]: cluster 2023-10-09T04:50:09.149986+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[79733]: audit 2023-10-09T04:50:09.157241+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[79733]: cluster 2023-10-09T04:50:09.157410+0000 mon.a (mon.0) 1563 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[79733]: cluster 2023-10-09T04:50:09.182633+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:10 smithi082 bash[79733]: cluster 2023-10-09T04:50:09.184635+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T04:50:10.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:10.627 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:10.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:10.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:10.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:10.891 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:10.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:10.993 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:11.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:11.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:11.163 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:11.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:11.306 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:50:11.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:11.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.157892+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.159442+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.160184+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.160475+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.160490+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.162130+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.163091+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.163765+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.169156+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.355520+0000 osd.6 (osd.6) 121 : cluster 0 9.1 scrub starts 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:11 smithi153 bash[59565]: cluster 2023-10-09T04:50:10.746163+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1159: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.157892+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.159442+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.160184+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.160475+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.160490+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.162130+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.163091+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.163765+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.169156+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.355520+0000 osd.6 (osd.6) 121 : cluster 0 9.1 scrub starts 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[77063]: cluster 2023-10-09T04:50:10.746163+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1159: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.157892+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.159442+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:50:11.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.160184+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.160475+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.160490+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.162130+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.163091+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.163765+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.169156+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.355520+0000 osd.6 (osd.6) 121 : cluster 0 9.1 scrub starts 2023-10-09T04:50:11.428 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:11 smithi082 bash[79733]: cluster 2023-10-09T04:50:10.746163+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1159: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:11.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:11.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:11.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:11.679 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:11.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:11.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:11.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:11.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:11.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:11.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:11.992 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:50:12.052 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:50:12.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:12.099 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:50:12.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:12.141 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:50:12.174 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:50:12.213 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:50:12.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.252 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.253 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:50:12.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:12.290 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:50:12.293 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.293 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.293 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.293 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.294 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.294 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.294 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.294 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:50:12.355 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:12 smithi153 bash[59565]: cluster 2023-10-09T04:50:11.422105+0000 osd.6 (osd.6) 122 : cluster 0 9.1 scrub ok 2023-10-09T04:50:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:12 smithi153 bash[59565]: cluster 2023-10-09T04:50:11.493793+0000 osd.5 (osd.5) 169 : cluster 0 9.7 scrub starts 2023-10-09T04:50:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:12 smithi153 bash[59565]: audit 2023-10-09T04:50:11.951586+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.82:0/2410774621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[77063]: cluster 2023-10-09T04:50:11.422105+0000 osd.6 (osd.6) 122 : cluster 0 9.1 scrub ok 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[77063]: cluster 2023-10-09T04:50:11.493793+0000 osd.5 (osd.5) 169 : cluster 0 9.7 scrub starts 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[77063]: audit 2023-10-09T04:50:11.951586+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.82:0/2410774621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[79733]: cluster 2023-10-09T04:50:11.422105+0000 osd.6 (osd.6) 122 : cluster 0 9.1 scrub ok 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[79733]: cluster 2023-10-09T04:50:11.493793+0000 osd.5 (osd.5) 169 : cluster 0 9.7 scrub starts 2023-10-09T04:50:12.470 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:12 smithi082 bash[79733]: audit 2023-10-09T04:50:11.951586+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.82:0/2410774621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:12.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:12.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:12.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:12.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:12.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:12.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:12.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:12.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:12.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:12.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:13.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:13.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:13.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:13.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:13.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:13.207 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:50:13.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:13.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:13.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:13.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:13 smithi153 bash[59565]: cluster 2023-10-09T04:50:12.140050+0000 osd.0 (osd.0) 3 : cluster 0 9.5 deep-scrub starts 2023-10-09T04:50:13.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:13 smithi153 bash[59565]: audit 2023-10-09T04:50:12.376204+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.82:0/307980377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:13 smithi153 bash[59565]: audit 2023-10-09T04:50:12.376801+0000 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.429 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:13 smithi153 bash[59565]: cluster 2023-10-09T04:50:12.746683+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1160: 99 pgs: 8 peering, 9 unknown, 82 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:13.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:13.566 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[77063]: cluster 2023-10-09T04:50:12.140050+0000 osd.0 (osd.0) 3 : cluster 0 9.5 deep-scrub starts 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[77063]: audit 2023-10-09T04:50:12.376204+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.82:0/307980377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[77063]: audit 2023-10-09T04:50:12.376801+0000 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[77063]: cluster 2023-10-09T04:50:12.746683+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1160: 99 pgs: 8 peering, 9 unknown, 82 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[79733]: cluster 2023-10-09T04:50:12.140050+0000 osd.0 (osd.0) 3 : cluster 0 9.5 deep-scrub starts 2023-10-09T04:50:13.577 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[79733]: audit 2023-10-09T04:50:12.376204+0000 mon.c (mon.2) 119 : audit 1 from='client.? 172.21.15.82:0/307980377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.578 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[79733]: audit 2023-10-09T04:50:12.376801+0000 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:13.578 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:13 smithi082 bash[79733]: cluster 2023-10-09T04:50:12.746683+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1160: 99 pgs: 8 peering, 9 unknown, 82 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2023-10-09T04:50:13.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:13.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:13.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:13.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:13.875 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:13.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:13.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:14.008 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:14.119 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:14.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:14.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.105563+0000 osd.0 (osd.0) 4 : cluster 0 9.9 scrub starts 2023-10-09T04:50:14.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.171380+0000 mon.a (mon.0) 1566 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:14.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.191037+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.192820+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.195884+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: audit 2023-10-09T04:50:13.196808+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.197273+0000 mon.a (mon.0) 1568 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.197607+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[77063]: cluster 2023-10-09T04:50:13.198235+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.105563+0000 osd.0 (osd.0) 4 : cluster 0 9.9 scrub starts 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.171380+0000 mon.a (mon.0) 1566 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:14.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.191037+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.192820+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.195884+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: audit 2023-10-09T04:50:13.196808+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.197273+0000 mon.a (mon.0) 1568 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.197607+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.185 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:14 smithi082 bash[79733]: cluster 2023-10-09T04:50:13.198235+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.254 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:14.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:14.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.105563+0000 osd.0 (osd.0) 4 : cluster 0 9.9 scrub starts 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.171380+0000 mon.a (mon.0) 1566 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.191037+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.192820+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.195884+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: audit 2023-10-09T04:50:13.196808+0000 mon.a (mon.0) 1567 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.197273+0000 mon.a (mon.0) 1568 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.197607+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.394 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:14 smithi153 bash[59565]: cluster 2023-10-09T04:50:13.198235+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T04:50:14.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:14.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:14.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:14.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:14.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:14.781 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:14.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:14.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:14.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:14.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:15.059 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:15.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:15.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:15.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:15.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:15.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:15.464 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:15.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[77063]: audit 2023-10-09T04:50:14.299448+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:15.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[77063]: audit 2023-10-09T04:50:14.300946+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:15.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[77063]: cluster 2023-10-09T04:50:14.747416+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1162: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:15.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[79733]: audit 2023-10-09T04:50:14.299448+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:15.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[79733]: audit 2023-10-09T04:50:14.300946+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:15.475 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:15 smithi082 bash[79733]: cluster 2023-10-09T04:50:14.747416+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1162: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:15.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:15.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:15.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:15.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:15 smithi153 bash[59565]: audit 2023-10-09T04:50:14.299448+0000 mon.a (mon.0) 1569 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:15.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:15 smithi153 bash[59565]: audit 2023-10-09T04:50:14.300946+0000 mon.a (mon.0) 1570 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:15.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:15 smithi153 bash[59565]: cluster 2023-10-09T04:50:14.747416+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1162: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:15.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:15.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:15.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:15.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:15.904 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:15.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:16.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:16.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:16.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:16.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:16.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:16.309 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:16.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:16.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:50:16.374 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:50:16.374 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:50:16.374 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:50:16.374 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T04:50:16.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T04:50:16.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:16.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:16.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:16.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:16.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:16.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:16.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:16.780 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:16.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:16.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:16.910 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:16.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:16.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:16.916 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:16 smithi082 bash[79733]: audit 2023-10-09T04:50:16.749872+0000 mon.a (mon.0) 1571 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:50:16.916 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:16 smithi082 bash[77063]: audit 2023-10-09T04:50:16.749872+0000 mon.a (mon.0) 1571 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:50:17.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:17.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:16 smithi153 bash[59565]: audit 2023-10-09T04:50:16.749872+0000 mon.a (mon.0) 1571 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:50:17.041 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:17.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:17.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:17.278 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:17.307 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:17.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:50:17.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:17.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:17.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:17.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:17.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:17.838 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:17.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:17.886 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:17.941 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.953 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: cluster 2023-10-09T04:50:16.748181+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1163: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:17.953 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: cluster 2023-10-09T04:50:16.808383+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:17.953 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: cluster 2023-10-09T04:50:16.811517+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:17.954 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: audit 2023-10-09T04:50:16.811982+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:50:17.954 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: cluster 2023-10-09T04:50:16.812055+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:50:17.954 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:17 smithi153 bash[59565]: cluster 2023-10-09T04:50:16.818942+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:17.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:17.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:18.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:18.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: cluster 2023-10-09T04:50:16.748181+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1163: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: cluster 2023-10-09T04:50:16.808383+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: cluster 2023-10-09T04:50:16.811517+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: audit 2023-10-09T04:50:16.811982+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: cluster 2023-10-09T04:50:16.812055+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[79733]: cluster 2023-10-09T04:50:16.818942+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: cluster 2023-10-09T04:50:16.748181+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1163: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: cluster 2023-10-09T04:50:16.808383+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: cluster 2023-10-09T04:50:16.811517+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: audit 2023-10-09T04:50:16.811982+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:50:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: cluster 2023-10-09T04:50:16.812055+0000 mon.a (mon.0) 1573 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:50:18.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:17 smithi082 bash[77063]: cluster 2023-10-09T04:50:16.818942+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T04:50:18.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:18.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:18.371 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:18.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:18.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:18.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:18.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:18.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:18.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:18.776 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:18.881 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.909 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:18.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:19.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.818107+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.025 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.818848+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.820003+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.820950+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.821125+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.823294+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: cluster 2023-10-09T04:50:17.824817+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.026 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:18 smithi153 bash[59565]: audit 2023-10-09T04:50:18.750647+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:50:19.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:19.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.818107+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.818848+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.820003+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.820950+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.821125+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.163 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.823294+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: cluster 2023-10-09T04:50:17.824817+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[79733]: audit 2023-10-09T04:50:18.750647+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.818107+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.818848+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.820003+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.820950+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.821125+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.164 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.823294+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.165 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: cluster 2023-10-09T04:50:17.824817+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T04:50:19.165 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:18 smithi082 bash[77063]: audit 2023-10-09T04:50:18.750647+0000 mon.a (mon.0) 1575 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:50:19.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:19.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:19.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:19.427 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:19.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:19.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:19.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:19.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:19.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:19.705 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:19] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T04:50:19.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.749043+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1166: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 1.45k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.820534+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: audit 2023-10-09T04:50:18.821713+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.821799+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.822431+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.823383+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.823464+0000 osd.1 (osd.1) 8 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.833578+0000 osd.1 (osd.1) 9 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.833 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[77063]: cluster 2023-10-09T04:50:18.840544+0000 osd.1 (osd.1) 10 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.749043+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1166: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 1.45k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.820534+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: audit 2023-10-09T04:50:18.821713+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.821799+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.822431+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.823383+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.823464+0000 osd.1 (osd.1) 8 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.833578+0000 osd.1 (osd.1) 9 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.834 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:19 smithi082 bash[79733]: cluster 2023-10-09T04:50:18.840544+0000 osd.1 (osd.1) 10 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.835 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:19.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:19.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.978 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.749043+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1166: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 35 MiB/s wr, 1.45k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:50:19.978 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.820534+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: audit 2023-10-09T04:50:18.821713+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.821799+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.822431+0000 osd.7 (osd.7) 196 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.823383+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.823464+0000 osd.1 (osd.1) 8 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.833578+0000 osd.1 (osd.1) 9 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:19.979 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:19 smithi153 bash[59565]: cluster 2023-10-09T04:50:18.840544+0000 osd.1 (osd.1) 10 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 260'7702 2023-10-09T04:50:20.000 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:20.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:20.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:20.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:20.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:20.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:20.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:20.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:20.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:20.804 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:20.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.143304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.838627+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.840577+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.844117+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.844485+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.845713+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.846970+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: cluster 2023-10-09T04:50:19.849745+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:20.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: audit 2023-10-09T04:50:19.936988+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:20.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:20 smithi153 bash[59565]: audit 2023-10-09T04:50:20.751005+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:50:20.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:20.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:21.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.143304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.838627+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.840577+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.844117+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.844485+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.845713+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.060 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.846970+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: cluster 2023-10-09T04:50:19.849745+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: audit 2023-10-09T04:50:19.936988+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[77063]: audit 2023-10-09T04:50:20.751005+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.143304+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.838627+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.840577+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.844117+0000 osd.7 (osd.7) 197 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.844485+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.061 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.845713+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.846970+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: cluster 2023-10-09T04:50:19.849745+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T04:50:21.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: audit 2023-10-09T04:50:19.936988+0000 mon.a (mon.0) 1579 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:21.062 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:20 smithi082 bash[79733]: audit 2023-10-09T04:50:20.751005+0000 mon.a (mon.0) 1580 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:50:21.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:21.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:21.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:21.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:21.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:21.360 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:21.362 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:21.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:21.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:21.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:21.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.729 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:21.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:21.860 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:21.950 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:21.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:21.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:21.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:22.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:22 smithi153 bash[59565]: cluster 2023-10-09T04:50:20.750012+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1169: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 8280/276288 objects misplaced (2.997%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:22.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:22 smithi153 bash[59565]: cluster 2023-10-09T04:50:21.051040+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T04:50:22.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:22 smithi153 bash[59565]: audit 2023-10-09T04:50:21.052987+0000 mon.a (mon.0) 1581 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:50:22.103 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:22 smithi153 bash[59565]: cluster 2023-10-09T04:50:21.053035+0000 mon.a (mon.0) 1582 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:50:22.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:22.226 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[77063]: cluster 2023-10-09T04:50:20.750012+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1169: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 8280/276288 objects misplaced (2.997%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[77063]: cluster 2023-10-09T04:50:21.051040+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T04:50:22.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[77063]: audit 2023-10-09T04:50:21.052987+0000 mon.a (mon.0) 1581 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:50:22.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[77063]: cluster 2023-10-09T04:50:21.053035+0000 mon.a (mon.0) 1582 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:50:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[79733]: cluster 2023-10-09T04:50:20.750012+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1169: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 8280/276288 objects misplaced (2.997%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[79733]: cluster 2023-10-09T04:50:21.051040+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T04:50:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[79733]: audit 2023-10-09T04:50:21.052987+0000 mon.a (mon.0) 1581 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:50:22.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:22 smithi082 bash[79733]: cluster 2023-10-09T04:50:21.053035+0000 mon.a (mon.0) 1582 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:22.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:22.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:22.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:22.512 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:22.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:22.645 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:22.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:22.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:22.908 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:22.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:23.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:23.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.061123+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.061615+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.062058+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.063622+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.070819+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.072463+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[77063]: audit 2023-10-09T04:50:22.752078+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.061123+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.061615+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.062058+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.063622+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.070819+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.072463+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.150 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:23 smithi082 bash[79733]: audit 2023-10-09T04:50:22.752078+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:50:23.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:23.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:23.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:23.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:23.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:23.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.061123+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.061615+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.062058+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.063622+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.070819+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.072463+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:23.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:23 smithi153 bash[59565]: audit 2023-10-09T04:50:22.752078+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:50:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:23.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:23.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:23.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:23.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:23.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:23.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:23.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:23.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:24.036 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:24.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:24.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:24.198 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:24.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:24.324 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.291571+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:22.750742+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1172: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 8280/274458 objects misplaced (3.017%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.067534+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.067541+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.069559+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: audit 2023-10-09T04:50:23.071105+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:50:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.071196+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:50:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.071718+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.071902+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:24 smithi153 bash[59565]: cluster 2023-10-09T04:50:23.080630+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.427 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:24.456 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.291571+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:22.750742+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1172: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 8280/274458 objects misplaced (3.017%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.067534+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.067541+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.069559+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: audit 2023-10-09T04:50:23.071105+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.071196+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.071718+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.071902+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[77063]: cluster 2023-10-09T04:50:23.080630+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.291571+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:22.750742+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1172: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 8280/274458 objects misplaced (3.017%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.067534+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.067541+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.069559+0000 osd.1 (osd.1) 11 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: audit 2023-10-09T04:50:23.071105+0000 mon.a (mon.0) 1585 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.071196+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.071718+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.071902+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T04:50:24.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:24 smithi082 bash[79733]: cluster 2023-10-09T04:50:23.080630+0000 osd.1 (osd.1) 12 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 264'7762 2023-10-09T04:50:24.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:24.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:24.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:24.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:24.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:24.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:24.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:24.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:24.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:25.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:25.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:25.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:25.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.069146+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.070376+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.072049+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.072894+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.076104+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[79733]: audit 2023-10-09T04:50:24.752668+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.069146+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.070376+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.072049+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.072894+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.076104+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:25 smithi082 bash[77063]: audit 2023-10-09T04:50:24.752668+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:50:25.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:25.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:25.345 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.069146+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.345 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.070376+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.345 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.072049+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:50:25.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.072894+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.076104+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T04:50:25.346 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:25 smithi153 bash[59565]: audit 2023-10-09T04:50:24.752668+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:50:25.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:25.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:25.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:25.630 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:25.734 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:25.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:25.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:25.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:25.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:26.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:26.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:26.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:26.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:26.397 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.406 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:24.751498+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1175: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8280/271062 objects misplaced (3.055%) 2023-10-09T04:50:26.406 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:25.084030+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:25.084045+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:25.086184+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: audit 2023-10-09T04:50:25.087224+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:25.087325+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: cluster 2023-10-09T04:50:25.088576+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[79733]: audit 2023-10-09T04:50:25.890793+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:24.751498+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1175: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8280/271062 objects misplaced (3.055%) 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:25.084030+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.407 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:25.084045+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:25.086184+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: audit 2023-10-09T04:50:25.087224+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:50:26.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:25.087325+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:50:26.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: cluster 2023-10-09T04:50:25.088576+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:26 smithi082 bash[77063]: audit 2023-10-09T04:50:25.890793+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:24.751498+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1175: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8280/271062 objects misplaced (3.055%) 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:25.084030+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:25.084045+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:25.086184+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: audit 2023-10-09T04:50:25.087224+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:25.087325+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:50:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: cluster 2023-10-09T04:50:25.088576+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T04:50:26.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:26 smithi153 bash[59565]: audit 2023-10-09T04:50:25.890793+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:26.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:26.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:26.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:26.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:26.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.837 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:26.941 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:26.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:26.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:27.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:27.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:27.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:27.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:27.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:27.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:27.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:27.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:27.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:27.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:27.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:27.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:27.733 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:27.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:27.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:27.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:27.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:27.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:27.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:28.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:28 smithi153 bash[59565]: cluster 2023-10-09T04:50:26.752183+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 15677/268833 objects misplaced (5.832%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:50:28.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.290 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:28.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:28.404 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:28 smithi082 bash[77063]: cluster 2023-10-09T04:50:26.752183+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 15677/268833 objects misplaced (5.832%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:50:28.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:28 smithi082 bash[79733]: cluster 2023-10-09T04:50:26.752183+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 15677/268833 objects misplaced (5.832%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:50:28.414 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:50:28.414 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:50:28.414 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:50:28.415 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:50:28.415 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:50:28.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:28.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:28.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:28.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:28.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:28.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.839 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:28.943 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.980 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:28.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:29.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:29.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:29.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:29.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:29.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:29.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:29.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:29.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:29.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:29.649 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:29.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:29.794 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:29.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:29.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:29.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:29.949 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:29] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T04:50:30.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:30.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:30.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:30.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:30 smithi082 bash[77063]: cluster 2023-10-09T04:50:28.756027+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 15677/268599 objects misplaced (5.837%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:50:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:30 smithi082 bash[77063]: audit 2023-10-09T04:50:29.292136+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:30 smithi082 bash[79733]: cluster 2023-10-09T04:50:28.756027+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 15677/268599 objects misplaced (5.837%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:50:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:30 smithi082 bash[79733]: audit 2023-10-09T04:50:29.292136+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:30.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:30.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:30.423 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:30 smithi153 bash[59565]: cluster 2023-10-09T04:50:28.756027+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 15677/268599 objects misplaced (5.837%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:50:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:30 smithi153 bash[59565]: audit 2023-10-09T04:50:29.292136+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:30.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:30.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:30.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:30.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:30.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:30.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:30.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:30.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:30.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:30.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:30.980 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:31.012 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:31.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:31.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:31.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:31.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:31.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:31.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:31.518 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:31.553 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:31.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:31.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:31.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:31.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:31.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:31.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:50:31.980 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:31.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:32.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:32.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:32.214 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:32.227 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:32 smithi082 bash[77063]: cluster 2023-10-09T04:50:30.756643+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1179: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 13681/261168 objects misplaced (5.238%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:50:32.227 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:32 smithi082 bash[79733]: cluster 2023-10-09T04:50:30.756643+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1179: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 13681/261168 objects misplaced (5.238%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:50:32.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:32.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:32.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:32 smithi153 bash[59565]: cluster 2023-10-09T04:50:30.756643+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1179: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 13681/261168 objects misplaced (5.238%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:50:32.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:32.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.682 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:32.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:32.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:32.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:32.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:33.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:33.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:33.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:33.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:33.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:33.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:33.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:33.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:33.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:33.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:33.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:33.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:33.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:33.938 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:33.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:33.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:34.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:34.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:34.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[77063]: cluster 2023-10-09T04:50:32.757165+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 13423/258735 objects misplaced (5.188%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:50:34.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[77063]: audit 2023-10-09T04:50:33.189788+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[77063]: audit 2023-10-09T04:50:33.199466+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[79733]: cluster 2023-10-09T04:50:32.757165+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 13423/258735 objects misplaced (5.188%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:50:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[79733]: audit 2023-10-09T04:50:33.189788+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:34 smithi082 bash[79733]: audit 2023-10-09T04:50:33.199466+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.263 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:34.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:34.378 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:34 smithi153 bash[59565]: cluster 2023-10-09T04:50:32.757165+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 13423/258735 objects misplaced (5.188%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:50:34.379 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:34 smithi153 bash[59565]: audit 2023-10-09T04:50:33.189788+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.379 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:34 smithi153 bash[59565]: audit 2023-10-09T04:50:33.199466+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:34.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:34.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:34.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:34.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:34.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:34.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:34.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:34.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:34.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.967 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:34.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:35.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:35.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:35.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:35.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:35.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:35.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:35.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:35.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:35.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:35.957 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:35.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:36.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.071 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:35 smithi082 bash[77063]: cluster 2023-10-09T04:50:34.757749+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1181: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 13423/252525 objects misplaced (5.316%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:50:36.071 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:35 smithi082 bash[77063]: audit 2023-10-09T04:50:34.953433+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:36.072 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:35 smithi082 bash[79733]: cluster 2023-10-09T04:50:34.757749+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1181: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 13423/252525 objects misplaced (5.316%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:50:36.072 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:35 smithi082 bash[79733]: audit 2023-10-09T04:50:34.953433+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:36.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:36.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:36.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:36.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:36.329 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:35 smithi153 bash[59565]: cluster 2023-10-09T04:50:34.757749+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1181: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 13423/252525 objects misplaced (5.316%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:50:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:35 smithi153 bash[59565]: audit 2023-10-09T04:50:34.953433+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:36.462 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:36.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:36.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:36.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:36.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:36.792 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:36.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:36.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:36.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:36.994 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:36 smithi082 bash[79733]: audit 2023-10-09T04:50:36.759930+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:50:36.996 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:36 smithi082 bash[77063]: audit 2023-10-09T04:50:36.759930+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:50:37.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:36 smithi153 bash[59565]: audit 2023-10-09T04:50:36.759930+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:50:37.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:37.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:37.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:37.626 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:37.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:37.729 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:37.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:37.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:37.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:37.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:37.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:37.983 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:38.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: cluster 2023-10-09T04:50:36.758471+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 11561/246921 objects misplaced (4.682%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:50:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: cluster 2023-10-09T04:50:36.958531+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: audit 2023-10-09T04:50:36.959784+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:50:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: cluster 2023-10-09T04:50:36.959856+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:50:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: cluster 2023-10-09T04:50:36.960963+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: audit 2023-10-09T04:50:37.623910+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:37 smithi153 bash[59565]: audit 2023-10-09T04:50:37.629134+0000 mon.a (mon.0) 1600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.242 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: cluster 2023-10-09T04:50:36.758471+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 11561/246921 objects misplaced (4.682%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: cluster 2023-10-09T04:50:36.958531+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: audit 2023-10-09T04:50:36.959784+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: cluster 2023-10-09T04:50:36.959856+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: cluster 2023-10-09T04:50:36.960963+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: audit 2023-10-09T04:50:37.623910+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[77063]: audit 2023-10-09T04:50:37.629134+0000 mon.a (mon.0) 1600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: cluster 2023-10-09T04:50:36.758471+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 11561/246921 objects misplaced (4.682%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: cluster 2023-10-09T04:50:36.958531+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: audit 2023-10-09T04:50:36.959784+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: cluster 2023-10-09T04:50:36.959856+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: cluster 2023-10-09T04:50:36.960963+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T04:50:38.298 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: audit 2023-10-09T04:50:37.623910+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.299 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:37 smithi082 bash[79733]: audit 2023-10-09T04:50:37.629134+0000 mon.a (mon.0) 1600 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:38.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:38.401 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:38.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.527 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:38.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:38.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:38.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:38.857 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:38.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:38.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:39.023 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:39.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:39.068 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:50:39.091 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:50:39.113 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:50:39.137 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:50:39.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:39.157 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:50:39.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: cluster 2023-10-09T04:50:37.960296+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.161 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: cluster 2023-10-09T04:50:37.962935+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:50:39.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: cluster 2023-10-09T04:50:37.963987+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: audit 2023-10-09T04:50:38.098387+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:39.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: audit 2023-10-09T04:50:38.099492+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:39.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: audit 2023-10-09T04:50:38.107019+0000 mon.a (mon.0) 1604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:39.162 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:38 smithi153 bash[59565]: audit 2023-10-09T04:50:38.760753+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:50:39.179 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:39.183 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:50:39.205 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:50:39.226 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:50:39.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.229 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.229 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.229 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.229 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:50:39.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: cluster 2023-10-09T04:50:37.960296+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: cluster 2023-10-09T04:50:37.962935+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:50:39.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: cluster 2023-10-09T04:50:37.963987+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: audit 2023-10-09T04:50:38.098387+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: audit 2023-10-09T04:50:38.099492+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: audit 2023-10-09T04:50:38.107019+0000 mon.a (mon.0) 1604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[79733]: audit 2023-10-09T04:50:38.760753+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: cluster 2023-10-09T04:50:37.960296+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: cluster 2023-10-09T04:50:37.962935+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: cluster 2023-10-09T04:50:37.963987+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: audit 2023-10-09T04:50:38.098387+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: audit 2023-10-09T04:50:38.099492+0000 mon.a (mon.0) 1603 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: audit 2023-10-09T04:50:38.107019+0000 mon.a (mon.0) 1604 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:39.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:38 smithi082 bash[77063]: audit 2023-10-09T04:50:38.760753+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:50:39.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:39.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:39.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:39.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:39.557 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:39.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:39.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:39.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:39.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:39.828 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:39] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T04:50:39.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:39.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:39.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:39.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:39.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:40.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:40.089 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.759297+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1185: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11561/246555 objects misplaced (4.689%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:50:40.089 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.968261+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.089 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: audit 2023-10-09T04:50:38.970268+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:50:40.090 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.970364+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:50:40.090 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.970968+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.090 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.972109+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.090 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: cluster 2023-10-09T04:50:38.974395+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.090 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[77063]: audit 2023-10-09T04:50:39.044698+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.82:0/28426466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.759297+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1185: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11561/246555 objects misplaced (4.689%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.968261+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: audit 2023-10-09T04:50:38.970268+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.970364+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.970968+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.972109+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: cluster 2023-10-09T04:50:38.974395+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.091 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:39 smithi082 bash[79733]: audit 2023-10-09T04:50:39.044698+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.82:0/28426466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:40.108 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:40.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:40.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:40.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:40.382 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:40.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.759297+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1185: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11561/246555 objects misplaced (4.689%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:50:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.968261+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: audit 2023-10-09T04:50:38.970268+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:50:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.970364+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:50:40.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.970968+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.972109+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: cluster 2023-10-09T04:50:38.974395+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T04:50:40.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:39 smithi153 bash[59565]: audit 2023-10-09T04:50:39.044698+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.82:0/28426466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:40.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:40.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:40.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:40.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:40.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:40.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:40.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:40.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:40.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:41.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[77063]: cluster 2023-10-09T04:50:39.989241+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[77063]: cluster 2023-10-09T04:50:39.990521+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[77063]: cluster 2023-10-09T04:50:39.994309+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[77063]: cluster 2023-10-09T04:50:39.997266+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[77063]: audit 2023-10-09T04:50:40.760684+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[79733]: cluster 2023-10-09T04:50:39.989241+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[79733]: cluster 2023-10-09T04:50:39.990521+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:50:41.036 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[79733]: cluster 2023-10-09T04:50:39.994309+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.037 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[79733]: cluster 2023-10-09T04:50:39.997266+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.037 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:40 smithi082 bash[79733]: audit 2023-10-09T04:50:40.760684+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:50:41.054 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:41.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:40 smithi153 bash[59565]: cluster 2023-10-09T04:50:39.989241+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:40 smithi153 bash[59565]: cluster 2023-10-09T04:50:39.990521+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:40 smithi153 bash[59565]: cluster 2023-10-09T04:50:39.994309+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:40 smithi153 bash[59565]: cluster 2023-10-09T04:50:39.997266+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:40 smithi153 bash[59565]: audit 2023-10-09T04:50:40.760684+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:50:41.190 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:41.293 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:41.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:41.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:41.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:41.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:41.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.719 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:41.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.859 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:41.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.989 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:41.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:41.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:42.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.103 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:42 smithi082 bash[79733]: cluster 2023-10-09T04:50:40.759910+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1188: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 10036/240906 objects misplaced (4.166%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:50:42.103 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:42 smithi082 bash[79733]: cluster 2023-10-09T04:50:40.999074+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.103 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:42 smithi082 bash[79733]: audit 2023-10-09T04:50:41.000601+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:50:42.103 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:42 smithi082 bash[79733]: cluster 2023-10-09T04:50:41.000690+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:50:42.103 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:42 smithi082 bash[79733]: cluster 2023-10-09T04:50:41.001526+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:41 smithi082 bash[77063]: cluster 2023-10-09T04:50:40.759910+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1188: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 10036/240906 objects misplaced (4.166%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:50:42.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:41 smithi082 bash[77063]: cluster 2023-10-09T04:50:40.999074+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:41 smithi082 bash[77063]: audit 2023-10-09T04:50:41.000601+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:50:42.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:41 smithi082 bash[77063]: cluster 2023-10-09T04:50:41.000690+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:50:42.104 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:41 smithi082 bash[77063]: cluster 2023-10-09T04:50:41.001526+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:42.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:42.243 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:42.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:42.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:42 smithi153 bash[59565]: cluster 2023-10-09T04:50:40.759910+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1188: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 97 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 10036/240906 objects misplaced (4.166%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:50:42.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:42 smithi153 bash[59565]: cluster 2023-10-09T04:50:40.999074+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:42 smithi153 bash[59565]: audit 2023-10-09T04:50:41.000601+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:50:42.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:42 smithi153 bash[59565]: cluster 2023-10-09T04:50:41.000690+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:50:42.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:42 smithi153 bash[59565]: cluster 2023-10-09T04:50:41.001526+0000 osd.4 (osd.4) 227 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T04:50:42.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:42.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:42.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:42.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:42.639 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:42.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:42.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.763 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:42.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:42.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:42.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:42.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:43.027 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:43.040 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T04:50:43.130 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:43.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.008668+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.011249+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.012617+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.014127+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.022536+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[79733]: audit 2023-10-09T04:50:42.350442+0000 mon.a (mon.0) 1613 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.008668+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.011249+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.012617+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.014127+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.022536+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:43 smithi082 bash[77063]: audit 2023-10-09T04:50:42.350442+0000 mon.a (mon.0) 1613 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:50:43.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.008668+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.011249+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.012617+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.014127+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.022536+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T04:50:43.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:43 smithi153 bash[59565]: audit 2023-10-09T04:50:42.350442+0000 mon.a (mon.0) 1613 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:50:43.285 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:43.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:43.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:43.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:43.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:43.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:43.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:43.664 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:43.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:43.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:43.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:43.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:44.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:44.153 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:44.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:44.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.298 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:42.760555+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1191: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6774/239355 objects degraded (2.830%); 9578/239355 objects misplaced (4.002%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.007328+0000 mon.a (mon.0) 1614 : cluster 3 Health check failed: Degraded data redundancy: 6774/239355 objects degraded (2.830%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.021168+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.021821+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.026105+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.026800+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: audit 2023-10-09T04:50:43.028891+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.028914+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.299 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:44 smithi153 bash[59565]: cluster 2023-10-09T04:50:43.028980+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:50:44.305 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:42.760555+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1191: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6774/239355 objects degraded (2.830%); 9578/239355 objects misplaced (4.002%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.007328+0000 mon.a (mon.0) 1614 : cluster 3 Health check failed: Degraded data redundancy: 6774/239355 objects degraded (2.830%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.021168+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.021821+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.026105+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.026800+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: audit 2023-10-09T04:50:43.028891+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.028914+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[77063]: cluster 2023-10-09T04:50:43.028980+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:42.760555+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1191: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6774/239355 objects degraded (2.830%); 9578/239355 objects misplaced (4.002%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:50:44.306 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.007328+0000 mon.a (mon.0) 1614 : cluster 3 Health check failed: Degraded data redundancy: 6774/239355 objects degraded (2.830%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.021168+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.021821+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.026105+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.026800+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: audit 2023-10-09T04:50:43.028891+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.82:0/4186422873' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.028914+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T04:50:44.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:44 smithi082 bash[79733]: cluster 2023-10-09T04:50:43.028980+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:50:44.327 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:44.430 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.456 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:44.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:44.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:44.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:44.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:44.831 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:44.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:44.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:44.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:45.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:45.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.020710+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.021978+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.022566+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.022937+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.023573+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.025477+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.026210+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.026820+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.028171+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.028703+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.029305+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.030567+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: audit 2023-10-09T04:50:44.296524+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: audit 2023-10-09T04:50:44.297693+0000 mon.a (mon.0) 1619 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:45.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:45 smithi153 bash[59565]: audit 2023-10-09T04:50:44.401050+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:45.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.020710+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.021978+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.022566+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.022937+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.023573+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.025477+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.026210+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.026820+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.028171+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.028703+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.029305+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.030567+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: audit 2023-10-09T04:50:44.296524+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: audit 2023-10-09T04:50:44.297693+0000 mon.a (mon.0) 1619 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[77063]: audit 2023-10-09T04:50:44.401050+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.020710+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.021978+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.022566+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.022937+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.023573+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.025477+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.026210+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.026820+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.028171+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.028703+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.029305+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.030567+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: audit 2023-10-09T04:50:44.296524+0000 mon.a (mon.0) 1618 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: audit 2023-10-09T04:50:44.297693+0000 mon.a (mon.0) 1619 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:45.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:45 smithi082 bash[79733]: audit 2023-10-09T04:50:44.401050+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:45.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:45.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:45.313 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:50:45.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:45.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:45.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:45.624 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:45.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:45.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:45.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:45.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:46.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:46.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:46.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:46.206 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:46.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:46.312 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:44.761244+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1194: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 92 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 6774/237126 objects degraded (2.857%); 9578/237126 objects misplaced (4.039%); 4.2 MiB/s, 63 objects/s recovering 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.293473+0000 mon.a (mon.0) 1621 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.294039+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.302182+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: audit 2023-10-09T04:50:45.303533+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.303732+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.308189+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[77063]: cluster 2023-10-09T04:50:45.315459+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:44.761244+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1194: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 92 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 6774/237126 objects degraded (2.857%); 9578/237126 objects misplaced (4.039%); 4.2 MiB/s, 63 objects/s recovering 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.293473+0000 mon.a (mon.0) 1621 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.294039+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.302182+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: audit 2023-10-09T04:50:45.303533+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.303732+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.308189+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.314 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:46 smithi082 bash[79733]: cluster 2023-10-09T04:50:45.315459+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:44.761244+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1194: 99 pgs: 4 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 92 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 6774/237126 objects degraded (2.857%); 9578/237126 objects misplaced (4.039%); 4.2 MiB/s, 63 objects/s recovering 2023-10-09T04:50:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.293473+0000 mon.a (mon.0) 1621 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.294039+0000 mon.a (mon.0) 1622 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.302182+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: audit 2023-10-09T04:50:45.303533+0000 mon.a (mon.0) 1623 : audit 1 from='client.? 172.21.15.82:0/1883201436' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.303732+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.308189+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:46 smithi153 bash[59565]: cluster 2023-10-09T04:50:45.315459+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T04:50:46.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:46.487 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:46.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:46.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:46.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:46.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:50:46.766 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:46.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:46.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:46.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:46.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:47.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:47.037 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:47.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:47.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:47.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:47.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:47.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:47.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:47.546 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:47.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:47.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:47.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:47.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:47.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:47.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:47.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:47.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:48.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:48.057 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:48 smithi082 bash[77063]: cluster 2023-10-09T04:50:46.762054+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1196: 99 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13393/233577 objects degraded (5.734%); 11198/233577 objects misplaced (4.794%); 8.6 MiB/s, 133 objects/s recovering 2023-10-09T04:50:48.057 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:48 smithi082 bash[79733]: cluster 2023-10-09T04:50:46.762054+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1196: 99 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13393/233577 objects degraded (5.734%); 11198/233577 objects misplaced (4.794%); 8.6 MiB/s, 133 objects/s recovering 2023-10-09T04:50:48.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:48.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:48.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:48 smithi153 bash[59565]: cluster 2023-10-09T04:50:46.762054+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1196: 99 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 13393/233577 objects degraded (5.734%); 11198/233577 objects misplaced (4.794%); 8.6 MiB/s, 133 objects/s recovering 2023-10-09T04:50:48.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:48.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:48.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:48.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:48.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:48.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:48.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:48.694 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:48.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:48.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:48.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:48.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:48.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:48.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:49.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:49.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:49.209 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:49 smithi082 bash[77063]: cluster 2023-10-09T04:50:49.054003+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 13393/232959 objects degraded (5.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:49.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:49 smithi082 bash[77063]: cluster 2023-10-09T04:50:49.054043+0000 mon.a (mon.0) 1626 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:50:49.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:49 smithi082 bash[79733]: cluster 2023-10-09T04:50:49.054003+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 13393/232959 objects degraded (5.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:49.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:49 smithi082 bash[79733]: cluster 2023-10-09T04:50:49.054043+0000 mon.a (mon.0) 1626 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:50:49.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:49.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:49.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:49.368 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:49 smithi153 bash[59565]: cluster 2023-10-09T04:50:49.054003+0000 mon.a (mon.0) 1625 : cluster 3 Health check update: Degraded data redundancy: 13393/232959 objects degraded (5.749%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:49.368 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:49 smithi153 bash[59565]: cluster 2023-10-09T04:50:49.054043+0000 mon.a (mon.0) 1626 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:50:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:49.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:49.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:49.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:49.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:49.733 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:49] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:50:49.734 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:49.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:49.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:49.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:49.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:49.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:50.028 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:50.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:50.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:50.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:50.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:50 smithi082 bash[77063]: cluster 2023-10-09T04:50:48.762624+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1197: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 13393/232959 objects degraded (5.749%); 11198/232959 objects misplaced (4.807%); 6.8 MiB/s, 105 objects/s recovering 2023-10-09T04:50:50.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:50 smithi082 bash[79733]: cluster 2023-10-09T04:50:48.762624+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1197: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 13393/232959 objects degraded (5.749%); 11198/232959 objects misplaced (4.807%); 6.8 MiB/s, 105 objects/s recovering 2023-10-09T04:50:50.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:50.280 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:50 smithi153 bash[59565]: cluster 2023-10-09T04:50:48.762624+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1197: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 13393/232959 objects degraded (5.749%); 11198/232959 objects misplaced (4.807%); 6.8 MiB/s, 105 objects/s recovering 2023-10-09T04:50:50.302 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:50.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:50.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:50.675 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:50.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:50.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:50.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:50.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:50.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:50.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:51.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:51.125 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:51.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:51.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:51.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:51.504 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:51.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:50:51.672 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:51.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:51.800 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:51.800 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:51.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:51.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:51.934 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:51.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:51.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:52.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:52.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:52.119 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:52 smithi082 bash[77063]: cluster 2023-10-09T04:50:50.763875+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1198: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 13393/225243 objects degraded (5.946%); 10689/225243 objects misplaced (4.746%); 7.3 MiB/s, 114 objects/s recovering 2023-10-09T04:50:52.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:52 smithi153 bash[59565]: cluster 2023-10-09T04:50:50.763875+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1198: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 13393/225243 objects degraded (5.946%); 10689/225243 objects misplaced (4.746%); 7.3 MiB/s, 114 objects/s recovering 2023-10-09T04:50:52.185 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:50:52.185 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:50:52.186 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:50:52.186 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T04:50:52.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:52.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:52.315 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:52 smithi082 bash[79733]: cluster 2023-10-09T04:50:50.763875+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1198: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 13393/225243 objects degraded (5.946%); 10689/225243 objects misplaced (4.746%); 7.3 MiB/s, 114 objects/s recovering 2023-10-09T04:50:52.339 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:52.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:52.475 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:52.577 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:52.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:52.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:52.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:52.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:52.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:52.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:52.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:53.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:53.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:53.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:53.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:53.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:53.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:50:53.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:53.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:53.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:53.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:53.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:53.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:53.801 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:53.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:53.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:53.941 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:53.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:53.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:54.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:54.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:54.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:54.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:54.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:54.381 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.385 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:54 smithi153 bash[59565]: cluster 2023-10-09T04:50:52.764508+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1199: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12927/222147 objects degraded (5.819%); 10375/222147 objects misplaced (4.670%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T04:50:54.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:54 smithi082 bash[77063]: cluster 2023-10-09T04:50:52.764508+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1199: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12927/222147 objects degraded (5.819%); 10375/222147 objects misplaced (4.670%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T04:50:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:54 smithi082 bash[79733]: cluster 2023-10-09T04:50:52.764508+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1199: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 12927/222147 objects degraded (5.819%); 10375/222147 objects misplaced (4.670%); 9.3 MiB/s, 146 objects/s recovering 2023-10-09T04:50:54.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:54.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:54.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:54.658 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:54.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:54.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:54.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:54.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:54.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:54.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:54.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:55.057 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:55.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:55.199 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:55.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:55 smithi082 bash[77063]: cluster 2023-10-09T04:50:55.051068+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 12927/222147 objects degraded (5.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:55.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:55 smithi082 bash[79733]: cluster 2023-10-09T04:50:55.051068+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 12927/222147 objects degraded (5.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:55.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:55.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:55.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:55.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:55 smithi153 bash[59565]: cluster 2023-10-09T04:50:55.051068+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: Degraded data redundancy: 12927/222147 objects degraded (5.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:50:55.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:55.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:55.606 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:55.648 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:55.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:55.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:55.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:55.888 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:55.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:55.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:56.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:56.063 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:56.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:56.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:56.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:56.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:56.309 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:56 smithi082 bash[77063]: cluster 2023-10-09T04:50:54.765131+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1200: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12927/219165 objects degraded (5.898%); 10375/219165 objects misplaced (4.734%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:50:56.310 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:56 smithi082 bash[79733]: cluster 2023-10-09T04:50:54.765131+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1200: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12927/219165 objects degraded (5.898%); 10375/219165 objects misplaced (4.734%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:50:56.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:56.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:56 smithi153 bash[59565]: cluster 2023-10-09T04:50:54.765131+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1200: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12927/219165 objects degraded (5.898%); 10375/219165 objects misplaced (4.734%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:50:56.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:56.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:56.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.608 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:56.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:56.709 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:56.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:56.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:56.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:56.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:57.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:57.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:57.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:57.182 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:57.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:57.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:57.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:57.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:57.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:57.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:57.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:57.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:57.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:57.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:57.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:57.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:57.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:58.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:58.025 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:58.026 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:58.028 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:58.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:58.138 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:50:58 smithi082 bash[77063]: cluster 2023-10-09T04:50:56.765988+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1201: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 12457/211098 objects degraded (5.901%); 9616/211098 objects misplaced (4.555%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:50:58.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:58.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:58.279 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:58.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:58.396 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:50:58 smithi082 bash[79733]: cluster 2023-10-09T04:50:56.765988+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1201: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 12457/211098 objects degraded (5.901%); 9616/211098 objects misplaced (4.555%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:50:58.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:50:58 smithi153 bash[59565]: cluster 2023-10-09T04:50:56.765988+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1201: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 12457/211098 objects degraded (5.901%); 9616/211098 objects misplaced (4.555%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:50:58.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:58.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:58.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:58.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:58.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:58.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:58.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:58.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:58.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:58.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:58.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:59.084 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.109 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:59.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:59.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:59.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:59.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:59.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:59.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.646 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:59.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:59.773 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:59.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:59.906 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:50:59.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:50:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:50:59] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:51:00.008 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:00.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:00.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:00.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:00.275 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:00.283 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[79733]: cluster 2023-10-09T04:50:58.766620+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1202: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 12457/210363 objects degraded (5.922%); 9616/210363 objects misplaced (4.571%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:51:00.283 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[79733]: audit 2023-10-09T04:50:59.292751+0000 mon.a (mon.0) 1628 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:00.283 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[79733]: cluster 2023-10-09T04:51:00.052439+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 12457/210363 objects degraded (5.922%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:00.284 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[77063]: cluster 2023-10-09T04:50:58.766620+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1202: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 12457/210363 objects degraded (5.922%); 9616/210363 objects misplaced (4.571%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:51:00.284 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[77063]: audit 2023-10-09T04:50:59.292751+0000 mon.a (mon.0) 1628 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:00.284 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:00 smithi082 bash[77063]: cluster 2023-10-09T04:51:00.052439+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 12457/210363 objects degraded (5.922%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:00.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:00.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:00 smithi153 bash[59565]: cluster 2023-10-09T04:50:58.766620+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1202: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 12457/210363 objects degraded (5.922%); 9616/210363 objects misplaced (4.571%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:51:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:00 smithi153 bash[59565]: audit 2023-10-09T04:50:59.292751+0000 mon.a (mon.0) 1628 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:00 smithi153 bash[59565]: cluster 2023-10-09T04:51:00.052439+0000 mon.a (mon.0) 1629 : cluster 3 Health check update: Degraded data redundancy: 12457/210363 objects degraded (5.922%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:00.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:00.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:00.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:00.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:00.705 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:00.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:00.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:00.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:00.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:00.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:00.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:01.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:01.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:01.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:01.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:01.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:01.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:01.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:01.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:01.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:01.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:01.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:01.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:01.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:01.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:01.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:01.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:01.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:01.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:02.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:02.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.138 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:02.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:02.252 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:02 smithi153 bash[59565]: cluster 2023-10-09T04:51:00.767736+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1203: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12457/202443 objects degraded (6.153%); 9095/202443 objects misplaced (4.493%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:51:02.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:02.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:02.450 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:02 smithi082 bash[77063]: cluster 2023-10-09T04:51:00.767736+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1203: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12457/202443 objects degraded (6.153%); 9095/202443 objects misplaced (4.493%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:51:02.450 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:02 smithi082 bash[79733]: cluster 2023-10-09T04:51:00.767736+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1203: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 12457/202443 objects degraded (6.153%); 9095/202443 objects misplaced (4.493%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:51:02.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:02.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:02.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:02.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:02.809 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:02.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:02.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:02.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:03.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:03.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:03.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:03.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:03.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:03.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:03.461 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:03 smithi082 bash[77063]: cluster 2023-10-09T04:51:02.768299+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1204: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 11983/199404 objects degraded (6.009%); 8785/199404 objects misplaced (4.406%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:51:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:03 smithi082 bash[79733]: cluster 2023-10-09T04:51:02.768299+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1204: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 11983/199404 objects degraded (6.009%); 8785/199404 objects misplaced (4.406%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:51:03.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:03.594 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:51:03.597 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:03.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:03.662 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:51:03.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:03 smithi153 bash[59565]: cluster 2023-10-09T04:51:02.768299+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1204: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 11983/199404 objects degraded (6.009%); 8785/199404 objects misplaced (4.406%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:51:03.697 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:51:03.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:03.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:51:03.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:03.773 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:51:03.807 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:51:03.845 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:51:03.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:03.885 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:51:03.886 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.886 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.886 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.886 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.886 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.887 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.887 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.887 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:51:03.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:03.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:04.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:04.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:04.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:04.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:04.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:04.256 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:04 smithi082 bash[77063]: audit 2023-10-09T04:51:03.558264+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.82:0/2591418927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:04.256 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:04 smithi082 bash[79733]: audit 2023-10-09T04:51:03.558264+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.82:0/2591418927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:04.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:04.298 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:04.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:04.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:04.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:04.554 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:04 smithi153 bash[59565]: audit 2023-10-09T04:51:03.558264+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.82:0/2591418927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:04.583 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:04.689 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:04.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:04.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:04.873 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:04.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:04.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:05.112 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:05.134 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:05.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:05.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:05.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:05.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:05.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:05 smithi082 bash[79733]: cluster 2023-10-09T04:51:04.769022+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1205: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 11983/196335 objects degraded (6.103%); 8785/196335 objects misplaced (4.474%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:51:05.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:05 smithi082 bash[79733]: cluster 2023-10-09T04:51:05.053324+0000 mon.a (mon.0) 1630 : cluster 3 Health check update: Degraded data redundancy: 11983/199404 objects degraded (6.009%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:05.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:05 smithi082 bash[77063]: cluster 2023-10-09T04:51:04.769022+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1205: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 11983/196335 objects degraded (6.103%); 8785/196335 objects misplaced (4.474%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:51:05.383 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:05 smithi082 bash[77063]: cluster 2023-10-09T04:51:05.053324+0000 mon.a (mon.0) 1630 : cluster 3 Health check update: Degraded data redundancy: 11983/199404 objects degraded (6.009%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:05.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:05 smithi153 bash[59565]: cluster 2023-10-09T04:51:04.769022+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1205: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 11983/196335 objects degraded (6.103%); 8785/196335 objects misplaced (4.474%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:51:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:05 smithi153 bash[59565]: cluster 2023-10-09T04:51:05.053324+0000 mon.a (mon.0) 1630 : cluster 3 Health check update: Degraded data redundancy: 11983/199404 objects degraded (6.009%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:05.503 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:05.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:05.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:05.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:05.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:05.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:05.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:06.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:06.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:06.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:06.153 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:06.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:06.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:06.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:06.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:06.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:06.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.591 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:06.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:06.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:06.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:06.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:06.954 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:06.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:06.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:07.093 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:07.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:07.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:07.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:07.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:07.403 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:07.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:07.646 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.680 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:07.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:07.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:07.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:07 smithi153 bash[59565]: cluster 2023-10-09T04:51:06.769843+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1206: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11563/188865 objects degraded (6.122%); 7962/188865 objects misplaced (4.216%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:51:07.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:07.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:08.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:08.098 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:08.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:08.120 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:07 smithi082 bash[79733]: cluster 2023-10-09T04:51:06.769843+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1206: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11563/188865 objects degraded (6.122%); 7962/188865 objects misplaced (4.216%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:51:08.120 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:07 smithi082 bash[77063]: cluster 2023-10-09T04:51:06.769843+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1206: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 11563/188865 objects degraded (6.122%); 7962/188865 objects misplaced (4.216%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:51:08.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:08.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:08.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:08.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:08.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:08.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:08.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:08.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:08.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:08.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:08.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:08.749 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:08.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:08.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:08.907 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:08.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:09.012 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:09.043 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:09.149 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:09.179 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:09.282 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:09.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:09.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:09.672 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:09.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:09.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:09.812 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:09] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:51:09.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:09.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:09.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:09.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:10.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:51:10.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:10.076 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:09 smithi082 bash[77063]: cluster 2023-10-09T04:51:08.770450+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1207: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 11563/188091 objects degraded (6.148%); 7962/188091 objects misplaced (4.233%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:51:10.076 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:09 smithi082 bash[79733]: cluster 2023-10-09T04:51:08.770450+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1207: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 11563/188091 objects degraded (6.148%); 7962/188091 objects misplaced (4.233%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:51:10.085 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:09 smithi153 bash[59565]: cluster 2023-10-09T04:51:08.770450+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1207: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 11563/188091 objects degraded (6.148%); 7962/188091 objects misplaced (4.233%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:51:10.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:10.211 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:10.243 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:10.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:10.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:10.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:10.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:10.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:10.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:10 smithi153 bash[59565]: cluster 2023-10-09T04:51:10.056281+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 11563/188091 objects degraded (6.148%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:10.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:10.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:11.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:11.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:10 smithi082 bash[77063]: cluster 2023-10-09T04:51:10.056281+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 11563/188091 objects degraded (6.148%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:11.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:10 smithi082 bash[79733]: cluster 2023-10-09T04:51:10.056281+0000 mon.a (mon.0) 1631 : cluster 3 Health check update: Degraded data redundancy: 11563/188091 objects degraded (6.148%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:11.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:11.316 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:11.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.451 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:11.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:11.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:11.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:11.721 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:11.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:11.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:11.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:11.984 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:11.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:12.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:12.098 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:12 smithi082 bash[77063]: cluster 2023-10-09T04:51:10.771172+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1208: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 11563/180390 objects degraded (6.410%); 7365/180390 objects misplaced (4.083%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:51:12.098 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:12 smithi082 bash[79733]: cluster 2023-10-09T04:51:10.771172+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1208: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 11563/180390 objects degraded (6.410%); 7365/180390 objects misplaced (4.083%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:51:12.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:12.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:12.223 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:12.246 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:12.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:12.377 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:12 smithi153 bash[59565]: cluster 2023-10-09T04:51:10.771172+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1208: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 11563/180390 objects degraded (6.410%); 7365/180390 objects misplaced (4.083%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:51:12.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:12.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:12.614 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:12.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:12.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:12.742 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:12.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:12.813 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:51:12.814 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:51:12.814 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.814 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161810 9.a e278/250 9:5f4e81b3:::benchmark_data_smithi082_200616_object52449:head v 278'10212, mlcod=278'10208)", 2023-10-09T04:51:12.814 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.793103+0000", 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.015320433, 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.015351485999999999, 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.815 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793103+0000", 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.816 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793103+0000", 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793102+0000", 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.817 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793115+0000", 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2527000000000001e-05 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.818 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793307+0000", 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000192444 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.819 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.793317+0000", 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0111e-05 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797032+0000", 2023-10-09T04:51:12.820 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0037148490000000001 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797293+0000", 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00026049500000000002 2023-10-09T04:51:12.821 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161820 9.f e278/250 9:ff554993:::benchmark_data_smithi082_200616_object52459:head v 278'10268, mlcod=278'10265)", 2023-10-09T04:51:12.822 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.797688+0000", 2023-10-09T04:51:12.823 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.010735915, 2023-10-09T04:51:12.823 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010839755, 2023-10-09T04:51:12.823 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.823 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.823 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797688+0000", 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797688+0000", 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.825 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797687+0000", 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.826 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797693+0000", 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9939999999999997e-06 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797702+0000", 2023-10-09T04:51:12.827 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0599999999999997e-06 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797707+0000", 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0390000000000004e-06 2023-10-09T04:51:12.828 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797916+0000", 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000209256 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.829 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.797981+0000", 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4678000000000007e-05 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.830 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161819 9.d e278/250 9:ba11605c:::benchmark_data_smithi082_200616_object52458:head v 278'9925, mlcod=278'9920)", 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.803672+0000", 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0047512759999999996, 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049209179999999998, 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.831 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803672+0000", 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.832 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803672+0000", 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803671+0000", 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.833 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803679+0000", 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1969999999999996e-06 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.834 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803691+0000", 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1770000000000001e-05 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803699+0000", 2023-10-09T04:51:12.835 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7300000000000005e-06 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803822+0000", 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000122934 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.836 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803874+0000", 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1632000000000003e-05 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.837 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161823 9.2 e278/276 9:421290ba:::benchmark_data_smithi082_200616_object52462:head v 278'8817, mlcod=278'8815)", 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.802099+0000", 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0063246520000000001, 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0065586719999999998, 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.802099+0000", 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.838 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.802099+0000", 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.839 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.841 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.841 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.802098+0000", 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.802106+0000", 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.435e-06 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.842 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803741+0000", 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0016353979999999999 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803747+0000", 2023-10-09T04:51:12.843 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.631e-06 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.805141+0000", 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0013945380000000001 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.844 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.805213+0000", 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2175000000000006e-05 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.845 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161822 9.d e278/250)", 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.803711+0000", 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0047129930000000004, 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0050089990000000001, 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.846 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803711+0000", 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803711+0000", 2023-10-09T04:51:12.847 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803710+0000", 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.848 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803714+0000", 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9029999999999997e-06 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803842+0000", 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012789 2023-10-09T04:51:12.849 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803852+0000", 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0332999999999999e-05 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.850 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161795 9.17 e278/275)", 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.803862+0000", 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0045618630000000002, 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0049061670000000003, 2023-10-09T04:51:12.851 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803862+0000", 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.852 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803862+0000", 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803861+0000", 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.853 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803872+0000", 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0363000000000001e-05 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.854 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803885+0000", 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2967e-05 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.803890+0000", 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5570000000000003e-06 2023-10-09T04:51:12.855 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161806 9.17 e278/275)", 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.807577+0000", 2023-10-09T04:51:12.856 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00084667900000000003, 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0012388880000000001, 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807577+0000", 2023-10-09T04:51:12.857 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807577+0000", 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.858 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807576+0000", 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807584+0000", 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4839999999999998e-06 2023-10-09T04:51:12.859 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807602+0000", 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8627999999999999e-05 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.860 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.807611+0000", 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8489999999999995e-06 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161801 9.d e278/250 9:b8952a17:::benchmark_data_smithi082_200616_object52440:head v 278'9922, mlcod=278'9920)", 2023-10-09T04:51:12.861 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.769691+0000", 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.038733018000000001, 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.039179802, 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.862 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769691+0000", 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769691+0000", 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.863 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769690+0000", 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.864 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769697+0000", 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2339999999999997e-06 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769706+0000", 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.3600000000000002e-06 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.865 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.769713+0000", 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2360000000000001e-06 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.774024+0000", 2023-10-09T04:51:12.866 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0043110240000000001 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.774166+0000", 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014262500000000001 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.867 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161807 9.d e278/250 9:b08358df:::benchmark_data_smithi082_200616_object52446:head v 278'9923, mlcod=278'9920)", 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.777418+0000", 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.031005995000000001, 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.031516836999999999, 2023-10-09T04:51:12.868 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777418+0000", 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.869 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777418+0000", 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777417+0000", 2023-10-09T04:51:12.870 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777422+0000", 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2809999999999999e-06 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777429+0000", 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3110000000000002e-06 2023-10-09T04:51:12.871 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.777434+0000", 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8810000000000004e-06 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.784086+0000", 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0066524660000000001 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.872 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.784105+0000", 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9049e-05 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161803 9.a e278/250 9:53670e7d:::benchmark_data_smithi082_200616_object52442:head v 278'10210, mlcod=278'10206)", 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.781160+0000", 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.027263164999999999, 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027839860000000001, 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.873 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781160+0000", 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781160+0000", 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.874 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781160+0000", 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781168+0000", 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6470000000000004e-06 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.875 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781180+0000", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2077e-05 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781198+0000", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7312999999999998e-05 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.782055+0000", 2023-10-09T04:51:12.876 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00085751999999999996 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.782111+0000", 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5931999999999999e-05 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161805 9.a e278/250 9:5153c13f:::benchmark_data_smithi082_200616_object52444:head v 278'10211, mlcod=278'10206)", 2023-10-09T04:51:12.877 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.781213+0000", 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.027210373, 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.027863763, 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781213+0000", 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.878 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781213+0000", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781213+0000", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781218+0000", 2023-10-09T04:51:12.879 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0799999999999996e-06 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781225+0000", 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.905e-06 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.781230+0000", 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6110000000000001e-06 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.880 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.792628+0000", 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.011397776 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.792652+0000", 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3920000000000001e-05 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.881 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.35685.0:161811 9.d e278/250 9:bd316392:::benchmark_data_smithi082_200616_object52450:head v 278'9924, mlcod=278'9920)", 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:12.785298+0000", 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.023125514999999999, 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023842123999999999, 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:51:12.882 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785298+0000", 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785298+0000", 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785297+0000", 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.883 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785305+0000", 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2030000000000003e-06 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785315+0000", 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0203e-05 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.884 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.785322+0000", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8669999999999996e-06 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.788111+0000", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027896610000000001 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:12.788168+0000", 2023-10-09T04:51:12.885 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6254e-05 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 12 2023-10-09T04:51:12.886 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:51:12.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:12.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:12.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:13.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:13.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:13 smithi153 bash[59565]: audit 2023-10-09T04:51:13.017559+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.82:0/257791161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:13 smithi153 bash[59565]: audit 2023-10-09T04:51:13.018127+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:13 smithi153 bash[59565]: audit 2023-10-09T04:51:13.024480+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.82:0/3447062869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:13 smithi153 bash[59565]: audit 2023-10-09T04:51:13.025119+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.092 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:13 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.025395+0000 mon.a (mon.0) 1634 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:51:13.103 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:51:13.106 INFO:teuthology.orchestra.run.smithi082.stderr:marked out osd.4. 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: debug 2023-10-09T04:51:13.023+0000 7f3fc98c6700 -1 mon.a@0(leader).osd e278 definitely_dead 0 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: audit 2023-10-09T04:51:13.017559+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.82:0/257791161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: audit 2023-10-09T04:51:13.018127+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: audit 2023-10-09T04:51:13.024480+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.82:0/3447062869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: audit 2023-10-09T04:51:13.025119+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.136 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.025395+0000 mon.a (mon.0) 1634 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:51:13.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[79733]: audit 2023-10-09T04:51:13.017559+0000 mon.c (mon.2) 122 : audit 1 from='client.? 172.21.15.82:0/257791161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[79733]: audit 2023-10-09T04:51:13.018127+0000 mon.a (mon.0) 1632 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[79733]: audit 2023-10-09T04:51:13.024480+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.82:0/3447062869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[79733]: audit 2023-10-09T04:51:13.025119+0000 mon.a (mon.0) 1633 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:51:13.137 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:13 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.025395+0000 mon.a (mon.0) 1634 : cluster 1 Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:51:13.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:13.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:13.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:13.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:13.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:13.558 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:13.580 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:13.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:13.704 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:13.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:13.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:13.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:13.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:13.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:14.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:14.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:14.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:12.771766+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1209: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 11123/177027 objects degraded (6.283%); 7045/177027 objects misplaced (3.980%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.075776+0000 mon.a (mon.0) 1635 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: audit 2023-10-09T04:51:13.094623+0000 mon.a (mon.0) 1636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: audit 2023-10-09T04:51:13.094759+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.094968+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.097705+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.098008+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.099646+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.099710+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.099935+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.101253+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.102212+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.105378+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:14 smithi153 bash[59565]: cluster 2023-10-09T04:51:13.108731+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:14.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:14.350 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:12.771766+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1209: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 11123/177027 objects degraded (6.283%); 7045/177027 objects misplaced (3.980%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:51:14.350 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.075776+0000 mon.a (mon.0) 1635 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: audit 2023-10-09T04:51:13.094623+0000 mon.a (mon.0) 1636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: audit 2023-10-09T04:51:13.094759+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.094968+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.097705+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.098008+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.099646+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.099710+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.099935+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.101253+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.102212+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.105378+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[77063]: cluster 2023-10-09T04:51:13.108731+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:12.771766+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1209: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 11123/177027 objects degraded (6.283%); 7045/177027 objects misplaced (3.980%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.075776+0000 mon.a (mon.0) 1635 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: audit 2023-10-09T04:51:13.094623+0000 mon.a (mon.0) 1636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: audit 2023-10-09T04:51:13.094759+0000 mon.a (mon.0) 1637 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.094968+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.097705+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.352 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.098008+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.099646+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.099710+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.099935+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.101253+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.102212+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.105378+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.353 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:14 smithi082 bash[79733]: cluster 2023-10-09T04:51:13.108731+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T04:51:14.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:14.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:14.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:14.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:14.591 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:14.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:14.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:14.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:14.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:14.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:14.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:14.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:14.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:15.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:15.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:15.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:15.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:15.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.093815+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.094054+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.097045+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T04:51:15.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.097120+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.097307+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.098108+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.100335+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.111170+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.114961+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: audit 2023-10-09T04:51:14.293012+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:15.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:15 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.057786+0000 mon.a (mon.0) 1641 : cluster 3 Health check update: Degraded data redundancy: 11123/177027 objects degraded (6.283%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:15.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:15.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:15.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.093815+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.094054+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.097045+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.097120+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.097307+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.098108+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.384 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.100335+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.111170+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.114961+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: audit 2023-10-09T04:51:14.293012+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.057786+0000 mon.a (mon.0) 1641 : cluster 3 Health check update: Degraded data redundancy: 11123/177027 objects degraded (6.283%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.093815+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.094054+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.097045+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.097120+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.097307+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.098108+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.385 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.100335+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.111170+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.114961+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T04:51:15.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: audit 2023-10-09T04:51:14.293012+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:15.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:15 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.057786+0000 mon.a (mon.0) 1641 : cluster 3 Health check update: Degraded data redundancy: 11123/177027 objects degraded (6.283%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:15.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:15.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:15.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:15.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:15.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:15.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:15.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:15.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:15.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:15.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:16.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:16.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:16.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:51:16.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:16.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:16.288 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:16.312 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:16.381 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.772737+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1212: 99 pgs: 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 11123/174465 objects degraded (6.375%); 7045/174465 objects misplaced (4.038%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:51:16.381 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:14.843471+0000 osd.2 (osd.2) 1 : cluster 0 9.2 scrub starts 2023-10-09T04:51:16.381 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.092224+0000 mon.a (mon.0) 1642 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:51:16.381 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.105533+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.381 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.107874+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.109981+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.110764+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.112693+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.114596+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.117289+0000 osd.7 (osd.7) 227 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 278'10029 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.117365+0000 osd.7 (osd.7) 228 : cluster 0 9.10 starting backfill to osd.2 from (253'100,271'8094] MIN to 278'8819 2023-10-09T04:51:16.382 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.118024+0000 osd.7 (osd.7) 229 : cluster 0 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 278'10033 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.772737+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1212: 99 pgs: 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 11123/174465 objects degraded (6.375%); 7045/174465 objects misplaced (4.038%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:14.843471+0000 osd.2 (osd.2) 1 : cluster 0 9.2 scrub starts 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.092224+0000 mon.a (mon.0) 1642 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.105533+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.107874+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.383 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.109981+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.110764+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.112693+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.114596+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.117289+0000 osd.7 (osd.7) 227 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 278'10029 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.117365+0000 osd.7 (osd.7) 228 : cluster 0 9.10 starting backfill to osd.2 from (253'100,271'8094] MIN to 278'8819 2023-10-09T04:51:16.384 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:16 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.118024+0000 osd.7 (osd.7) 229 : cluster 0 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 278'10033 2023-10-09T04:51:16.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.772737+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1212: 99 pgs: 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 11123/174465 objects degraded (6.375%); 7045/174465 objects misplaced (4.038%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:51:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:14.843471+0000 osd.2 (osd.2) 1 : cluster 0 9.2 scrub starts 2023-10-09T04:51:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.092224+0000 mon.a (mon.0) 1642 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:51:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.105533+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.107874+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.109981+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.110764+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.112693+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.114596+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.117289+0000 osd.7 (osd.7) 227 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 278'10029 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.117365+0000 osd.7 (osd.7) 228 : cluster 0 9.10 starting backfill to osd.2 from (253'100,271'8094] MIN to 278'8819 2023-10-09T04:51:16.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:16 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.118024+0000 osd.7 (osd.7) 229 : cluster 0 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 278'10033 2023-10-09T04:51:16.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:16.568 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:16.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:16.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:16.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:16.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:16.859 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:16.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:16.987 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:16.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:16.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:17.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:17.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:17.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:17.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:17.357 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:17.368 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:17 smithi153 bash[59565]: cluster 2023-10-09T04:51:15.244223+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:17.385 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:17.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:17 smithi082 bash[79733]: cluster 2023-10-09T04:51:15.244223+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:17.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:17 smithi082 bash[77063]: cluster 2023-10-09T04:51:15.244223+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T04:51:17.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:17.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:17.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:17.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:17.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:17.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:17.885 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:17.893 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:17.911 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:17.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:17.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:18.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:18.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:18.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:18.153 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:18 smithi082 bash[77063]: cluster 2023-10-09T04:51:16.773403+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1214: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 81 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8850/170274 objects degraded (5.198%); 16588/170274 objects misplaced (9.742%); 7.3 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-09T04:51:18.154 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:18 smithi082 bash[79733]: cluster 2023-10-09T04:51:16.773403+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1214: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 81 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8850/170274 objects degraded (5.198%); 16588/170274 objects misplaced (9.742%); 7.3 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-09T04:51:18.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:18.273 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:18.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:18.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:18.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:51:18.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:18.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:18 smithi153 bash[59565]: cluster 2023-10-09T04:51:16.773403+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1214: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 81 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8850/170274 objects degraded (5.198%); 16588/170274 objects misplaced (9.742%); 7.3 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-09T04:51:18.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:18.529 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:18.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:18.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:18.685 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:18.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:18.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:18.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:18.920 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:18.951 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:18.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:19.054 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:19.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:19.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:19.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:19.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:19.322 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:19 smithi082 bash[77063]: cluster 2023-10-09T04:51:18.773992+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1215: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 12226/169815 objects degraded (7.200%); 15775/169815 objects misplaced (9.290%); 5.2 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T04:51:19.323 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:19 smithi082 bash[77063]: cluster 2023-10-09T04:51:19.112691+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:51:19.323 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:19 smithi082 bash[79733]: cluster 2023-10-09T04:51:18.773992+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1215: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 12226/169815 objects degraded (7.200%); 15775/169815 objects misplaced (9.290%); 5.2 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T04:51:19.323 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:19 smithi082 bash[79733]: cluster 2023-10-09T04:51:19.112691+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:51:19.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:19.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:19 smithi153 bash[59565]: cluster 2023-10-09T04:51:18.773992+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1215: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 12226/169815 objects degraded (7.200%); 15775/169815 objects misplaced (9.290%); 5.2 MiB/s, 1 keys/s, 85 objects/s recovering 2023-10-09T04:51:19.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:19 smithi153 bash[59565]: cluster 2023-10-09T04:51:19.112691+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:51:19.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:19.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:19.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:19.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:19.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:19.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:19.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:19.835 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:19] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T04:51:19.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:19.952 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:19.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:19.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:20.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:20.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:20.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:20.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:20.381 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:20 smithi153 bash[59565]: cluster 2023-10-09T04:51:20.059416+0000 mon.a (mon.0) 1645 : cluster 3 Health check update: Degraded data redundancy: 12226/169815 objects degraded (7.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:20 smithi082 bash[77063]: cluster 2023-10-09T04:51:20.059416+0000 mon.a (mon.0) 1645 : cluster 3 Health check update: Degraded data redundancy: 12226/169815 objects degraded (7.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:20.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:20 smithi082 bash[79733]: cluster 2023-10-09T04:51:20.059416+0000 mon.a (mon.0) 1645 : cluster 3 Health check update: Degraded data redundancy: 12226/169815 objects degraded (7.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:20.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:20.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:20.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:20.623 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:20.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.793 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:20.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:20.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:20.924 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:20.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:20.948 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:21.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:21.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:21.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:21.178 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:21.201 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:21.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:21.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:21.402 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:51:21.402 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:51:21.402 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T04:51:21.402 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:51:21.402 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:51:21.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T04:51:21.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:21.449 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:21 smithi153 bash[59565]: cluster 2023-10-09T04:51:20.774648+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1216: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 15611/169470 objects degraded (9.212%); 17122/169470 objects misplaced (10.103%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:21.468 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:21.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:21.600 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:21.710 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:21.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:21 smithi082 bash[79733]: cluster 2023-10-09T04:51:20.774648+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1216: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 15611/169470 objects degraded (9.212%); 17122/169470 objects misplaced (10.103%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:21.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:21 smithi082 bash[77063]: cluster 2023-10-09T04:51:20.774648+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1216: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 15611/169470 objects degraded (9.212%); 17122/169470 objects misplaced (10.103%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:21.744 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:21.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:21.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:21.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:21.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:22.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:22.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:22.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:22.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:22.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:22.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:22.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:22.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:22.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:22 smithi082 bash[79733]: cluster 2023-10-09T04:51:21.337057+0000 osd.6 (osd.6) 138 : cluster 0 9.0 scrub starts 2023-10-09T04:51:22.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:22 smithi082 bash[77063]: cluster 2023-10-09T04:51:21.337057+0000 osd.6 (osd.6) 138 : cluster 0 9.0 scrub starts 2023-10-09T04:51:22.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:22.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:22.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:22.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:22 smithi153 bash[59565]: cluster 2023-10-09T04:51:21.337057+0000 osd.6 (osd.6) 138 : cluster 0 9.0 scrub starts 2023-10-09T04:51:22.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:22.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:22.791 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:22.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:22.921 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:22.956 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:22.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:23.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:23.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:23.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:23.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:23.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:23.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:23.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:23 smithi082 bash[79733]: cluster 2023-10-09T04:51:22.777873+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1217: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 15545/168066 objects degraded (9.249%); 16807/168066 objects misplaced (10.000%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:23.331 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:23 smithi082 bash[77063]: cluster 2023-10-09T04:51:22.777873+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1217: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 15545/168066 objects degraded (9.249%); 16807/168066 objects misplaced (10.000%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:23.367 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:23.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:23.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:23.615 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:23.657 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:23 smithi153 bash[59565]: cluster 2023-10-09T04:51:22.777873+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1217: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 15545/168066 objects degraded (9.249%); 16807/168066 objects misplaced (10.000%); 7.5 MiB/s, 1 keys/s, 122 objects/s recovering 2023-10-09T04:51:23.763 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:23.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:23.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:23.933 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:23.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:23.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:24.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:24.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:24.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:24.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:24.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:24 smithi082 bash[79733]: cluster 2023-10-09T04:51:23.305484+0000 osd.6 (osd.6) 139 : cluster 0 9.0 scrub starts 2023-10-09T04:51:24.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:24 smithi082 bash[77063]: cluster 2023-10-09T04:51:23.305484+0000 osd.6 (osd.6) 139 : cluster 0 9.0 scrub starts 2023-10-09T04:51:24.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:24.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:24.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:24.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:24.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:24.615 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:24 smithi153 bash[59565]: cluster 2023-10-09T04:51:23.305484+0000 osd.6 (osd.6) 139 : cluster 0 9.0 scrub starts 2023-10-09T04:51:24.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:24.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:24.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:24.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:24.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:24.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:24.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:25.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:25.065 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:25.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:25.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:25.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:25.309 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:25 smithi153 bash[59565]: cluster 2023-10-09T04:51:24.778587+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1218: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 15545/165300 objects degraded (9.404%); 16807/165300 objects misplaced (10.168%); 6.5 MiB/s, 0 keys/s, 106 objects/s recovering 2023-10-09T04:51:25.310 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:25 smithi153 bash[59565]: cluster 2023-10-09T04:51:25.060392+0000 mon.a (mon.0) 1646 : cluster 3 Health check update: Degraded data redundancy: 15545/168066 objects degraded (9.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:25.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:25.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:25.481 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:25.581 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:25 smithi082 bash[77063]: cluster 2023-10-09T04:51:24.778587+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1218: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 15545/165300 objects degraded (9.404%); 16807/165300 objects misplaced (10.168%); 6.5 MiB/s, 0 keys/s, 106 objects/s recovering 2023-10-09T04:51:25.749 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:25 smithi082 bash[77063]: cluster 2023-10-09T04:51:25.060392+0000 mon.a (mon.0) 1646 : cluster 3 Health check update: Degraded data redundancy: 15545/168066 objects degraded (9.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:25.750 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:25 smithi082 bash[79733]: cluster 2023-10-09T04:51:24.778587+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1218: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 15545/165300 objects degraded (9.404%); 16807/165300 objects misplaced (10.168%); 6.5 MiB/s, 0 keys/s, 106 objects/s recovering 2023-10-09T04:51:25.750 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:25 smithi082 bash[79733]: cluster 2023-10-09T04:51:25.060392+0000 mon.a (mon.0) 1646 : cluster 3 Health check update: Degraded data redundancy: 15545/168066 objects degraded (9.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:51:25.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:25.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:25.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:25.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:25.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:26.027 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:26.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:26.061 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:26.169 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:26.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:26.214 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:26 smithi082 bash[77063]: cluster 2023-10-09T04:51:25.277646+0000 osd.6 (osd.6) 140 : cluster 0 9.0 scrub starts 2023-10-09T04:51:26.214 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:26 smithi082 bash[79733]: cluster 2023-10-09T04:51:25.277646+0000 osd.6 (osd.6) 140 : cluster 0 9.0 scrub starts 2023-10-09T04:51:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:26.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:26.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:26 smithi153 bash[59565]: cluster 2023-10-09T04:51:25.277646+0000 osd.6 (osd.6) 140 : cluster 0 9.0 scrub starts 2023-10-09T04:51:26.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:26.443 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:26.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:26.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:26.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:26.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:26.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:26.850 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:26.887 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:26.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:26.991 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:27.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:27.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:27.158 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:27.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:27.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:27.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:27.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:27 smithi153 bash[59565]: cluster 2023-10-09T04:51:26.232107+0000 osd.6 (osd.6) 141 : cluster 0 9.0 scrub starts 2023-10-09T04:51:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:27 smithi153 bash[59565]: cluster 2023-10-09T04:51:26.779345+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1219: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 15293/158037 objects degraded (9.677%); 15617/158037 objects misplaced (9.882%); 7.8 MiB/s, 0 keys/s, 125 objects/s recovering 2023-10-09T04:51:27.437 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:27.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:27 smithi082 bash[77063]: cluster 2023-10-09T04:51:26.232107+0000 osd.6 (osd.6) 141 : cluster 0 9.0 scrub starts 2023-10-09T04:51:27.518 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:27 smithi082 bash[77063]: cluster 2023-10-09T04:51:26.779345+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1219: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 15293/158037 objects degraded (9.677%); 15617/158037 objects misplaced (9.882%); 7.8 MiB/s, 0 keys/s, 125 objects/s recovering 2023-10-09T04:51:27.518 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:27 smithi082 bash[79733]: cluster 2023-10-09T04:51:26.232107+0000 osd.6 (osd.6) 141 : cluster 0 9.0 scrub starts 2023-10-09T04:51:27.518 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:27 smithi082 bash[79733]: cluster 2023-10-09T04:51:26.779345+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1219: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 15293/158037 objects degraded (9.677%); 15617/158037 objects misplaced (9.882%); 7.8 MiB/s, 0 keys/s, 125 objects/s recovering 2023-10-09T04:51:27.540 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:27.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:27.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:27.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:27.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:27.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:27.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:27.960 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:27.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:28.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:28.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:28.200 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:28.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:28.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:28.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:28.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:28.480 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:28 smithi153 bash[59565]: cluster 2023-10-09T04:51:27.277318+0000 osd.6 (osd.6) 142 : cluster 0 9.0 scrub starts 2023-10-09T04:51:28.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:28 smithi082 bash[77063]: cluster 2023-10-09T04:51:27.277318+0000 osd.6 (osd.6) 142 : cluster 0 9.0 scrub starts 2023-10-09T04:51:28.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:28 smithi082 bash[79733]: cluster 2023-10-09T04:51:27.277318+0000 osd.6 (osd.6) 142 : cluster 0 9.0 scrub starts 2023-10-09T04:51:28.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:28.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:28.630 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:28.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:28.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:28.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:28.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:28.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:28.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:28.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:29.029 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:29.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:29.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:29.162 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:29.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:29 smithi082 bash[77063]: cluster 2023-10-09T04:51:28.779899+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1220: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 15293/156597 objects degraded (9.766%); 15617/156597 objects misplaced (9.973%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:51:29.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:29.273 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:29 smithi082 bash[79733]: cluster 2023-10-09T04:51:28.779899+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1220: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 15293/156597 objects degraded (9.766%); 15617/156597 objects misplaced (9.973%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:51:29.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:29.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:29.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:29.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:29.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:29.546 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:29 smithi153 bash[59565]: cluster 2023-10-09T04:51:28.779899+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1220: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 15293/156597 objects degraded (9.766%); 15617/156597 objects misplaced (9.973%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:51:29.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:29.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:29.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:29.817 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:29.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:29.946 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:29.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:29.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:30.069 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:29] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-10-09T04:51:30.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:30.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:30.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.243 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:30.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:30.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:30.488 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:30 smithi153 bash[59565]: audit 2023-10-09T04:51:29.293293+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:30.489 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:30 smithi153 bash[59565]: cluster 2023-10-09T04:51:30.061694+0000 mon.a (mon.0) 1648 : cluster 3 Health check update: Degraded data redundancy: 15293/156597 objects degraded (9.766%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:30.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:30.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:30 smithi082 bash[77063]: audit 2023-10-09T04:51:29.293293+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:30 smithi082 bash[77063]: cluster 2023-10-09T04:51:30.061694+0000 mon.a (mon.0) 1648 : cluster 3 Health check update: Degraded data redundancy: 15293/156597 objects degraded (9.766%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:30 smithi082 bash[79733]: audit 2023-10-09T04:51:29.293293+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:30.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:30 smithi082 bash[79733]: cluster 2023-10-09T04:51:30.061694+0000 mon.a (mon.0) 1648 : cluster 3 Health check update: Degraded data redundancy: 15293/156597 objects degraded (9.766%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:30.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:30.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:30.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:30.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:30.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:30.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:31.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:31.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:31.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:31.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:31.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:31.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:31.416 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:31.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:31.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:31 smithi082 bash[77063]: cluster 2023-10-09T04:51:30.780683+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1221: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 15293/150729 objects degraded (10.146%); 15167/150729 objects misplaced (10.062%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T04:51:31.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:31 smithi082 bash[79733]: cluster 2023-10-09T04:51:30.780683+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1221: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 15293/150729 objects degraded (10.146%); 15167/150729 objects misplaced (10.062%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T04:51:31.554 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:31.563 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:31 smithi153 bash[59565]: cluster 2023-10-09T04:51:30.780683+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1221: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 15293/150729 objects degraded (10.146%); 15167/150729 objects misplaced (10.062%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T04:51:31.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:31.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:31.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:31.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:31.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:31.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:31.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:31.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:32.096 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.125 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:32.230 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:32.373 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:32.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:32.520 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:32.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.664 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:32.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:32.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:32.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:32.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:51:33.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:33.076 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:51:33.106 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:51:33.156 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:51:33.171 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.195 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:51:33.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:33.223 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.228 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.229 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:51:33.306 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:33.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:33.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:33.434 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:33.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:33.583 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:33.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:33.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:33.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:33.841 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:33 smithi082 bash[77063]: cluster 2023-10-09T04:51:32.781388+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1222: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 15079/146052 objects degraded (10.324%); 14447/146052 objects misplaced (9.892%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:51:33.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:33.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:33.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:33.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:34.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:34.130 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:34.139 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:33 smithi082 bash[77063]: audit 2023-10-09T04:51:32.899388+0000 mon.a (mon.0) 1649 : audit 0 from='client.? 172.21.15.82:0/1467412689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:34.139 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:33 smithi082 bash[79733]: cluster 2023-10-09T04:51:32.781388+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1222: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 15079/146052 objects degraded (10.324%); 14447/146052 objects misplaced (9.892%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:51:34.139 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:33 smithi082 bash[79733]: audit 2023-10-09T04:51:32.899388+0000 mon.a (mon.0) 1649 : audit 0 from='client.? 172.21.15.82:0/1467412689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:34.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:33 smithi153 bash[59565]: cluster 2023-10-09T04:51:32.781388+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1222: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 15079/146052 objects degraded (10.324%); 14447/146052 objects misplaced (9.892%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:51:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:33 smithi153 bash[59565]: audit 2023-10-09T04:51:32.899388+0000 mon.a (mon.0) 1649 : audit 0 from='client.? 172.21.15.82:0/1467412689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:34.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:34.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:34.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:34.455 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:34.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:34.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:34.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:34.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:34.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:34.849 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:34 smithi153 bash[59565]: cluster 2023-10-09T04:51:33.345296+0000 osd.6 (osd.6) 143 : cluster 0 9.0 scrub starts 2023-10-09T04:51:34.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:34.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:34.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:34.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:34 smithi082 bash[77063]: cluster 2023-10-09T04:51:33.345296+0000 osd.6 (osd.6) 143 : cluster 0 9.0 scrub starts 2023-10-09T04:51:34.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:34 smithi082 bash[79733]: cluster 2023-10-09T04:51:33.345296+0000 osd.6 (osd.6) 143 : cluster 0 9.0 scrub starts 2023-10-09T04:51:35.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:35.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:35.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:35.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:35.384 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:35.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:35.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:35.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:35.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:35.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:35.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:35.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:36.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:36.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:35 smithi153 bash[59565]: cluster 2023-10-09T04:51:34.782055+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1223: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 15079/143238 objects degraded (10.527%); 14447/143238 objects misplaced (10.086%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:51:36.066 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:35 smithi153 bash[59565]: cluster 2023-10-09T04:51:35.066501+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 15079/143238 objects degraded (10.527%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:36.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:36.119 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:35 smithi082 bash[77063]: cluster 2023-10-09T04:51:34.782055+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1223: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 15079/143238 objects degraded (10.527%); 14447/143238 objects misplaced (10.086%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:51:36.119 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:35 smithi082 bash[77063]: cluster 2023-10-09T04:51:35.066501+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 15079/143238 objects degraded (10.527%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:36.119 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:35 smithi082 bash[79733]: cluster 2023-10-09T04:51:34.782055+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1223: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 15079/143238 objects degraded (10.527%); 14447/143238 objects misplaced (10.086%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:51:36.119 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:35 smithi082 bash[79733]: cluster 2023-10-09T04:51:35.066501+0000 mon.a (mon.0) 1650 : cluster 3 Health check update: Degraded data redundancy: 15079/143238 objects degraded (10.527%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:36.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:36.227 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:36.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:36.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:36.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:36.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:36.497 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:36.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:36.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:36.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:36.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:36.765 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:36.872 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:36.906 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:36.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:37.015 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:37.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:37.185 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:37.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.316 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:37.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.443 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:37.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:37.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:37.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:37.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:37.810 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:37.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:37.940 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:37.949 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:37 smithi153 bash[59565]: cluster 2023-10-09T04:51:36.782789+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1224: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 14847/136425 objects degraded (10.883%); 13299/136425 objects misplaced (9.748%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:51:37.975 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:37.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:38.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.086 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:37 smithi082 bash[77063]: cluster 2023-10-09T04:51:36.782789+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1224: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 14847/136425 objects degraded (10.883%); 13299/136425 objects misplaced (9.748%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:51:38.086 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:37 smithi082 bash[79733]: cluster 2023-10-09T04:51:36.782789+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1224: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 14847/136425 objects degraded (10.883%); 13299/136425 objects misplaced (9.748%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:51:38.107 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:38.213 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:38.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:38.365 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:38.394 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:38.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:38.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:38.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:38.680 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:38.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:38.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:38.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:38 smithi153 bash[59565]: cluster 2023-10-09T04:51:37.445029+0000 osd.6 (osd.6) 144 : cluster 0 9.0 scrub starts 2023-10-09T04:51:38.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:38 smithi153 bash[59565]: audit 2023-10-09T04:51:38.215484+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:38.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:38.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:39.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:39.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:39.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:39.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:39.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:39.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:39.387 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:38 smithi082 bash[79733]: cluster 2023-10-09T04:51:37.445029+0000 osd.6 (osd.6) 144 : cluster 0 9.0 scrub starts 2023-10-09T04:51:39.388 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:38 smithi082 bash[79733]: audit 2023-10-09T04:51:38.215484+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:39.388 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:38 smithi082 bash[77063]: cluster 2023-10-09T04:51:37.445029+0000 osd.6 (osd.6) 144 : cluster 0 9.0 scrub starts 2023-10-09T04:51:39.388 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:38 smithi082 bash[77063]: audit 2023-10-09T04:51:38.215484+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:39.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:39.502 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:39.607 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:39.643 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:39.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:39.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:39.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:39 smithi153 bash[59565]: cluster 2023-10-09T04:51:38.464358+0000 osd.6 (osd.6) 145 : cluster 0 9.0 scrub starts 2023-10-09T04:51:39.868 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:39 smithi153 bash[59565]: cluster 2023-10-09T04:51:38.783435+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1225: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14847/135048 objects degraded (10.994%); 13299/135048 objects misplaced (9.848%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:51:39.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:39.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:39.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:40.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:40.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:40.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[77063]: cluster 2023-10-09T04:51:38.464358+0000 osd.6 (osd.6) 145 : cluster 0 9.0 scrub starts 2023-10-09T04:51:40.149 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[77063]: cluster 2023-10-09T04:51:38.783435+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1225: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14847/135048 objects degraded (10.994%); 13299/135048 objects misplaced (9.848%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:51:40.149 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:39] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-10-09T04:51:40.149 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[79733]: cluster 2023-10-09T04:51:38.464358+0000 osd.6 (osd.6) 145 : cluster 0 9.0 scrub starts 2023-10-09T04:51:40.149 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[79733]: cluster 2023-10-09T04:51:38.783435+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1225: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14847/135048 objects degraded (10.994%); 13299/135048 objects misplaced (9.848%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:51:40.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:40.269 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:40.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:40.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:40.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:40.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:40.486 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:51:40.486 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:51:40.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:40.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:40.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:40.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:40.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:40.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:40.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:40.927 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:40.970 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[77063]: cluster 2023-10-09T04:51:40.067374+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 14847/135048 objects degraded (10.994%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:40.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:40.971 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:40 smithi082 bash[79733]: cluster 2023-10-09T04:51:40.067374+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 14847/135048 objects degraded (10.994%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:41.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:41.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:41.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:40 smithi153 bash[59565]: cluster 2023-10-09T04:51:40.067374+0000 mon.a (mon.0) 1652 : cluster 3 Health check update: Degraded data redundancy: 14847/135048 objects degraded (10.994%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:51:41.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:41.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:41.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:41.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:41.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:41.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:51:41.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:41.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:41.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:41.905 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.935 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:41.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:42.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:42.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:42.552 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:42 smithi082 bash[79733]: cluster 2023-10-09T04:51:40.784108+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1226: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 14847/129567 objects degraded (11.459%); 12824/129567 objects misplaced (9.898%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:42.553 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:42 smithi082 bash[77063]: cluster 2023-10-09T04:51:40.784108+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1226: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 14847/129567 objects degraded (11.459%); 12824/129567 objects misplaced (9.898%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:42.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:42.554 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:42 smithi153 bash[59565]: cluster 2023-10-09T04:51:40.784108+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1226: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 14847/129567 objects degraded (11.459%); 12824/129567 objects misplaced (9.898%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:42.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:42.680 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:42.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:42.785 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:42.803 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:42.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:42.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:42.938 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:42.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:43.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:43.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:43.172 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:43.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:43.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:43.340 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:43.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:43.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:43.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:43.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:43.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:43.763 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:43.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:51:43.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:43.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:43.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:43.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:44.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:44.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:44.267 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.276 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:44 smithi153 bash[59565]: cluster 2023-10-09T04:51:42.528301+0000 osd.6 (osd.6) 146 : cluster 0 9.0 scrub starts 2023-10-09T04:51:44.276 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:44 smithi153 bash[59565]: cluster 2023-10-09T04:51:42.784670+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1227: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 14591/124761 objects degraded (11.695%); 12108/124761 objects misplaced (9.705%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:44.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:44.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:44.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:44.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:44 smithi082 bash[77063]: cluster 2023-10-09T04:51:42.528301+0000 osd.6 (osd.6) 146 : cluster 0 9.0 scrub starts 2023-10-09T04:51:44.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:44 smithi082 bash[77063]: cluster 2023-10-09T04:51:42.784670+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1227: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 14591/124761 objects degraded (11.695%); 12108/124761 objects misplaced (9.705%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:44.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:44 smithi082 bash[79733]: cluster 2023-10-09T04:51:42.528301+0000 osd.6 (osd.6) 146 : cluster 0 9.0 scrub starts 2023-10-09T04:51:44.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:44 smithi082 bash[79733]: cluster 2023-10-09T04:51:42.784670+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1227: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 14591/124761 objects degraded (11.695%); 12108/124761 objects misplaced (9.705%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:51:44.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:44.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:44.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.714 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:44.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:44.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:44.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:44.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:44.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:45.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:45.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:45.177 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:45.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:45.284 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:45.307 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[77063]: cluster 2023-10-09T04:51:43.528578+0000 osd.6 (osd.6) 147 : cluster 0 9.0 scrub starts 2023-10-09T04:51:45.308 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[77063]: audit 2023-10-09T04:51:44.293729+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:45.308 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[77063]: cluster 2023-10-09T04:51:45.068419+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 14591/124761 objects degraded (11.695%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:45.308 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[79733]: cluster 2023-10-09T04:51:43.528578+0000 osd.6 (osd.6) 147 : cluster 0 9.0 scrub starts 2023-10-09T04:51:45.308 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[79733]: audit 2023-10-09T04:51:44.293729+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:45.308 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:45 smithi082 bash[79733]: cluster 2023-10-09T04:51:45.068419+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 14591/124761 objects degraded (11.695%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:45.335 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:45.360 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:45 smithi153 bash[59565]: cluster 2023-10-09T04:51:43.528578+0000 osd.6 (osd.6) 147 : cluster 0 9.0 scrub starts 2023-10-09T04:51:45.360 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:45 smithi153 bash[59565]: audit 2023-10-09T04:51:44.293729+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:45.360 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:45 smithi153 bash[59565]: cluster 2023-10-09T04:51:45.068419+0000 mon.a (mon.0) 1654 : cluster 3 Health check update: Degraded data redundancy: 14591/124761 objects degraded (11.695%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:45.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:45.504 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:45.531 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:51:45.532 INFO:teuthology.orchestra.run.smithi082.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:51:45.532 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:51:45.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:45.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:45.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:45.765 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:45.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:45.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:45.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:46.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:46.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:46.104 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:51:46.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:46.275 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:46.283 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:46 smithi153 bash[59565]: cluster 2023-10-09T04:51:44.501124+0000 osd.6 (osd.6) 148 : cluster 0 9.0 scrub starts 2023-10-09T04:51:46.283 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:46 smithi153 bash[59565]: cluster 2023-10-09T04:51:44.785473+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1228: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14591/121584 objects degraded (12.001%); 12108/121584 objects misplaced (9.959%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:51:46.283 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:46 smithi153 bash[59565]: audit 2023-10-09T04:51:45.841761+0000 mon.a (mon.0) 1655 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:46.283 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:46 smithi153 bash[59565]: audit 2023-10-09T04:51:46.035153+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.283 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:46 smithi153 bash[59565]: audit 2023-10-09T04:51:46.047194+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.322 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:46.424 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.433 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[77063]: cluster 2023-10-09T04:51:44.501124+0000 osd.6 (osd.6) 148 : cluster 0 9.0 scrub starts 2023-10-09T04:51:46.433 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[77063]: cluster 2023-10-09T04:51:44.785473+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1228: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14591/121584 objects degraded (12.001%); 12108/121584 objects misplaced (9.959%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:51:46.433 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[77063]: audit 2023-10-09T04:51:45.841761+0000 mon.a (mon.0) 1655 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[77063]: audit 2023-10-09T04:51:46.035153+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[77063]: audit 2023-10-09T04:51:46.047194+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[79733]: cluster 2023-10-09T04:51:44.501124+0000 osd.6 (osd.6) 148 : cluster 0 9.0 scrub starts 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[79733]: cluster 2023-10-09T04:51:44.785473+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1228: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 88 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14591/121584 objects degraded (12.001%); 12108/121584 objects misplaced (9.959%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[79733]: audit 2023-10-09T04:51:45.841761+0000 mon.a (mon.0) 1655 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[79733]: audit 2023-10-09T04:51:46.035153+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.434 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:46 smithi082 bash[79733]: audit 2023-10-09T04:51:46.047194+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:46.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:46.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:46.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:46.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:46.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:46.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:46.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.875 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:46.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:46.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:46.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:47.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:47.129 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:47.173 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.092587+0000 mon.a (mon.0) 1658 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.095116+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.096210+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: audit 2023-10-09T04:51:46.097508+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.097580+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.099365+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.122003+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.092587+0000 mon.a (mon.0) 1658 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.095116+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.096210+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: audit 2023-10-09T04:51:46.097508+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.097580+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T04:51:47.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.099365+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:47 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.122003+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:47.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:47.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.092587+0000 mon.a (mon.0) 1658 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.095116+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.096210+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: audit 2023-10-09T04:51:46.097508+0000 mon.a (mon.0) 1659 : audit 1 from='client.? 172.21.15.82:0/2482691507' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.097580+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-09T04:51:47.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.099365+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:47 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.122003+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T04:51:47.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:47.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:47.576 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:47.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:47.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:47.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:47.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:47.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:47.974 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:47.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:48.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:48.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:48.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:48.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.354 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.509581+0000 osd.6 (osd.6) 150 : cluster 0 9.0 scrub starts 2023-10-09T04:51:48.354 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[77063]: cluster 2023-10-09T04:51:46.786106+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1230: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 14373/114669 objects degraded (12.534%); 10870/114669 objects misplaced (9.479%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:48.354 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[77063]: audit 2023-10-09T04:51:47.105459+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.354 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[77063]: audit 2023-10-09T04:51:47.132152+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.354 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.509581+0000 osd.6 (osd.6) 150 : cluster 0 9.0 scrub starts 2023-10-09T04:51:48.355 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[79733]: cluster 2023-10-09T04:51:46.786106+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1230: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 14373/114669 objects degraded (12.534%); 10870/114669 objects misplaced (9.479%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:48.355 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[79733]: audit 2023-10-09T04:51:47.105459+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.355 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:48 smithi082 bash[79733]: audit 2023-10-09T04:51:47.132152+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:48 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.509581+0000 osd.6 (osd.6) 150 : cluster 0 9.0 scrub starts 2023-10-09T04:51:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:48 smithi153 bash[59565]: cluster 2023-10-09T04:51:46.786106+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1230: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 14373/114669 objects degraded (12.534%); 10870/114669 objects misplaced (9.479%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:48 smithi153 bash[59565]: audit 2023-10-09T04:51:47.105459+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:48 smithi153 bash[59565]: audit 2023-10-09T04:51:47.132152+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:48.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:48.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:48.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:48.768 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:48.854 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:48.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:48.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:48.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:49.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:49.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:49.152 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:49.196 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:49.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:49.334 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:49.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:49.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:49.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:49.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:49.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:49.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:49.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:49.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:49.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:49.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:49] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-10-09T04:51:50.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:50.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:50.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:50.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:50.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:50.287 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:50 smithi082 bash[77063]: cluster 2023-10-09T04:51:48.786807+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1231: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 14373/113439 objects degraded (12.670%); 10870/113439 objects misplaced (9.582%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:50.288 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:50 smithi082 bash[77063]: cluster 2023-10-09T04:51:50.069734+0000 mon.a (mon.0) 1663 : cluster 3 Health check update: Degraded data redundancy: 14373/113439 objects degraded (12.670%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:50.288 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:50 smithi082 bash[79733]: cluster 2023-10-09T04:51:48.786807+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1231: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 14373/113439 objects degraded (12.670%); 10870/113439 objects misplaced (9.582%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:50.288 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:50 smithi082 bash[79733]: cluster 2023-10-09T04:51:50.069734+0000 mon.a (mon.0) 1663 : cluster 3 Health check update: Degraded data redundancy: 14373/113439 objects degraded (12.670%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:50.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:50 smithi153 bash[59565]: cluster 2023-10-09T04:51:48.786807+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1231: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 14373/113439 objects degraded (12.670%); 10870/113439 objects misplaced (9.582%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:51:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:50 smithi153 bash[59565]: cluster 2023-10-09T04:51:50.069734+0000 mon.a (mon.0) 1663 : cluster 3 Health check update: Degraded data redundancy: 14373/113439 objects degraded (12.670%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:50.430 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:50.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:51:50.457 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:50.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:50.585 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:50.690 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:50.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:50.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:50.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:50.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:50.955 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:50.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:51.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:51.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:51.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:51.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:51.295 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:51.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:51.402 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:51.432 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:51.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[77063]: audit 2023-10-09T04:51:50.283995+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[77063]: audit 2023-10-09T04:51:50.294992+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[77063]: cluster 2023-10-09T04:51:50.787479+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1232: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 14373/108426 objects degraded (13.256%); 10429/108426 objects misplaced (9.619%); 5.6 MiB/s, 95 objects/s recovering 2023-10-09T04:51:51.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[79733]: audit 2023-10-09T04:51:50.283995+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[79733]: audit 2023-10-09T04:51:50.294992+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:51 smithi082 bash[79733]: cluster 2023-10-09T04:51:50.787479+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1232: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 14373/108426 objects degraded (13.256%); 10429/108426 objects misplaced (9.619%); 5.6 MiB/s, 95 objects/s recovering 2023-10-09T04:51:51.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.544 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:51 smithi153 bash[59565]: audit 2023-10-09T04:51:50.283995+0000 mon.a (mon.0) 1664 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.544 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:51 smithi153 bash[59565]: audit 2023-10-09T04:51:50.294992+0000 mon.a (mon.0) 1665 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:51:51.544 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:51 smithi153 bash[59565]: cluster 2023-10-09T04:51:50.787479+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1232: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 14373/108426 objects degraded (13.256%); 10429/108426 objects misplaced (9.619%); 5.6 MiB/s, 95 objects/s recovering 2023-10-09T04:51:51.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:51.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:51.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:51.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:51.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:51.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:51.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:52.031 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:51:52.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:51:52.101 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:52.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:52.240 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:52.263 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:52.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:52.392 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:52.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:52.529 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:52.634 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:52.669 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:52.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:52.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:52.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:52.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:52.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:53.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:53.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:53.158 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:53.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:53.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:53.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:53.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:53.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:53.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:53.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:53.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:53.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:53.844 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:53 smithi082 bash[77063]: cluster 2023-10-09T04:51:52.788161+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1233: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13962/103992 objects degraded (13.426%); 9713/103992 objects misplaced (9.340%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:53.851 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:53.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:53.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:53.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:54.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:54.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:54.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:54.123 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:53 smithi153 bash[59565]: cluster 2023-10-09T04:51:52.788161+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1233: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13962/103992 objects degraded (13.426%); 9713/103992 objects misplaced (9.340%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:54.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:54.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:54.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:53 smithi082 bash[79733]: cluster 2023-10-09T04:51:52.788161+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1233: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13962/103992 objects degraded (13.426%); 9713/103992 objects misplaced (9.340%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:54.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:54.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:54.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:54.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:54.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:54.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:54.658 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:54.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:54.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:54.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:54.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:54.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:54.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:54.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:55.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:55.076 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:54 smithi082 bash[77063]: cluster 2023-10-09T04:51:53.511911+0000 osd.6 (osd.6) 151 : cluster 0 9.0 scrub starts 2023-10-09T04:51:55.077 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:54 smithi082 bash[79733]: cluster 2023-10-09T04:51:53.511911+0000 osd.6 (osd.6) 151 : cluster 0 9.0 scrub starts 2023-10-09T04:51:55.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:55.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:54 smithi153 bash[59565]: cluster 2023-10-09T04:51:53.511911+0000 osd.6 (osd.6) 151 : cluster 0 9.0 scrub starts 2023-10-09T04:51:55.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:55.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:55.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.364 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:55.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:55.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:55.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:55.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:55.765 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:55.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:55.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:55.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:55.988 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:55 smithi082 bash[77063]: cluster 2023-10-09T04:51:54.788792+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1234: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 13962/101085 objects degraded (13.812%); 9713/101085 objects misplaced (9.609%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:55.988 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:55 smithi082 bash[77063]: cluster 2023-10-09T04:51:55.071138+0000 mon.a (mon.0) 1666 : cluster 3 Health check update: Degraded data redundancy: 13962/101085 objects degraded (13.812%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:55.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:55 smithi082 bash[79733]: cluster 2023-10-09T04:51:54.788792+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1234: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 13962/101085 objects degraded (13.812%); 9713/101085 objects misplaced (9.609%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:55.988 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:55 smithi082 bash[79733]: cluster 2023-10-09T04:51:55.071138+0000 mon.a (mon.0) 1666 : cluster 3 Health check update: Degraded data redundancy: 13962/101085 objects degraded (13.812%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:55.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:56.046 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:56.048 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:56.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:55 smithi153 bash[59565]: cluster 2023-10-09T04:51:54.788792+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1234: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 13962/101085 objects degraded (13.812%); 9713/101085 objects misplaced (9.609%); 6.2 MiB/s, 105 objects/s recovering 2023-10-09T04:51:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:55 smithi153 bash[59565]: cluster 2023-10-09T04:51:55.071138+0000 mon.a (mon.0) 1666 : cluster 3 Health check update: Degraded data redundancy: 13962/101085 objects degraded (13.812%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:51:56.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:56.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:56.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.438 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:56.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:56.573 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:56.675 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:56.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:56.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:56.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.829 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:56.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:56.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:56.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:57.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:57.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:57.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:57.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.352 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:57.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:57.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:57.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:57.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:57.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:57.807 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:51:57.847 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:51:57.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.886 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:51:57.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:57.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:57.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:57.919 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:51:57.948 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[77063]: cluster 2023-10-09T04:51:56.541147+0000 osd.6 (osd.6) 152 : cluster 0 9.0 scrub starts 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[77063]: cluster 2023-10-09T04:51:56.789705+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1235: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 13541/93876 objects degraded (14.424%); 8708/93876 objects misplaced (9.276%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[77063]: audit 2023-10-09T04:51:57.756552+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.82:0/4287416637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[79733]: cluster 2023-10-09T04:51:56.541147+0000 osd.6 (osd.6) 152 : cluster 0 9.0 scrub starts 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[79733]: cluster 2023-10-09T04:51:56.789705+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1235: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 13541/93876 objects degraded (14.424%); 8708/93876 objects misplaced (9.276%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T04:51:57.986 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:57 smithi082 bash[79733]: audit 2023-10-09T04:51:57.756552+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.82:0/4287416637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:57.988 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:51:58.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:58.029 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:51:58.029 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:58.077 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.083 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.084 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:51:58.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:58.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:58.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:57 smithi153 bash[59565]: cluster 2023-10-09T04:51:56.541147+0000 osd.6 (osd.6) 152 : cluster 0 9.0 scrub starts 2023-10-09T04:51:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:57 smithi153 bash[59565]: cluster 2023-10-09T04:51:56.789705+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1235: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 13541/93876 objects degraded (14.424%); 8708/93876 objects misplaced (9.276%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T04:51:58.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:57 smithi153 bash[59565]: audit 2023-10-09T04:51:57.756552+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.82:0/4287416637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:58.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:58.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:58.402 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:58.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:58.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:58.554 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:58.658 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:58.682 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:58.785 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:58.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:58.922 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:58.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:58.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:59.050 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.082 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:59.190 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.215 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:59.321 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:59.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:59.485 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:59.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:59.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:59.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:59.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:59.740 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:51:59] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-10-09T04:51:59.750 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:59.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:59.889 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:51:59.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:59.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.031 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:00.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:00.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:59 smithi153 bash[59565]: cluster 2023-10-09T04:51:58.790728+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1236: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 13541/92595 objects degraded (14.624%); 8708/92595 objects misplaced (9.404%); 4.3 MiB/s, 95 objects/s recovering 2023-10-09T04:52:00.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:59 smithi153 bash[59565]: audit 2023-10-09T04:51:59.299813+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:00.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:51:59 smithi153 bash[59565]: audit 2023-10-09T04:51:59.301191+0000 mon.a (mon.0) 1668 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:00.166 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:00.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:00.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:00.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:00.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:00.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[77063]: cluster 2023-10-09T04:51:58.790728+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1236: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 13541/92595 objects degraded (14.624%); 8708/92595 objects misplaced (9.404%); 4.3 MiB/s, 95 objects/s recovering 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[77063]: audit 2023-10-09T04:51:59.299813+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[77063]: audit 2023-10-09T04:51:59.301191+0000 mon.a (mon.0) 1668 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[79733]: cluster 2023-10-09T04:51:58.790728+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1236: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 13541/92595 objects degraded (14.624%); 8708/92595 objects misplaced (9.404%); 4.3 MiB/s, 95 objects/s recovering 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[79733]: audit 2023-10-09T04:51:59.299813+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:00.489 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:51:59 smithi082 bash[79733]: audit 2023-10-09T04:51:59.301191+0000 mon.a (mon.0) 1668 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:00.527 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:00.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:00.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:00.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:00.961 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:00.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:01.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:01.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:00 smithi153 bash[59565]: cluster 2023-10-09T04:52:00.072432+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: Degraded data redundancy: 13541/92595 objects degraded (14.624%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:01.193 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:01.222 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:01.265 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:00 smithi082 bash[77063]: cluster 2023-10-09T04:52:00.072432+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: Degraded data redundancy: 13541/92595 objects degraded (14.624%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:01.265 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:00 smithi082 bash[79733]: cluster 2023-10-09T04:52:00.072432+0000 mon.a (mon.0) 1669 : cluster 3 Health check update: Degraded data redundancy: 13541/92595 objects degraded (14.624%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:01.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:01.354 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:01.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:01.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:01.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:01.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:01.629 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:01.734 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:01.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:01.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:01.912 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:01.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:02.016 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.037 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:02.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:02.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:02.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:02 smithi153 bash[59565]: cluster 2023-10-09T04:52:00.586144+0000 osd.6 (osd.6) 153 : cluster 0 9.0 scrub starts 2023-10-09T04:52:02.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:02 smithi153 bash[59565]: cluster 2023-10-09T04:52:00.791563+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1237: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 13541/87675 objects degraded (15.445%); 8494/87675 objects misplaced (9.688%); 4.3 MiB/s, 119 objects/s recovering 2023-10-09T04:52:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:02 smithi082 bash[77063]: cluster 2023-10-09T04:52:00.586144+0000 osd.6 (osd.6) 153 : cluster 0 9.0 scrub starts 2023-10-09T04:52:02.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:02 smithi082 bash[77063]: cluster 2023-10-09T04:52:00.791563+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1237: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 13541/87675 objects degraded (15.445%); 8494/87675 objects misplaced (9.688%); 4.3 MiB/s, 119 objects/s recovering 2023-10-09T04:52:02.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:02 smithi082 bash[79733]: cluster 2023-10-09T04:52:00.586144+0000 osd.6 (osd.6) 153 : cluster 0 9.0 scrub starts 2023-10-09T04:52:02.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:02 smithi082 bash[79733]: cluster 2023-10-09T04:52:00.791563+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1237: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 13541/87675 objects degraded (15.445%); 8494/87675 objects misplaced (9.688%); 4.3 MiB/s, 119 objects/s recovering 2023-10-09T04:52:02.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:02.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.447 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:02.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:02.588 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:02.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:02.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:02.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:02.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:02.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:03.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:03.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:03.214 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:03.223 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: cluster 2023-10-09T04:52:01.630258+0000 osd.6 (osd.6) 154 : cluster 0 9.0 scrub starts 2023-10-09T04:52:03.223 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: audit 2023-10-09T04:52:02.535352+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.224 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: audit 2023-10-09T04:52:02.566934+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.224 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: audit 2023-10-09T04:52:02.568432+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:03.224 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: audit 2023-10-09T04:52:02.573299+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:03.224 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:03 smithi153 bash[59565]: audit 2023-10-09T04:52:02.592465+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.246 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:03.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:03.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:03.480 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:03.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: cluster 2023-10-09T04:52:01.630258+0000 osd.6 (osd.6) 154 : cluster 0 9.0 scrub starts 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: audit 2023-10-09T04:52:02.535352+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: audit 2023-10-09T04:52:02.566934+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: audit 2023-10-09T04:52:02.568432+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: audit 2023-10-09T04:52:02.573299+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[77063]: audit 2023-10-09T04:52:02.592465+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: cluster 2023-10-09T04:52:01.630258+0000 osd.6 (osd.6) 154 : cluster 0 9.0 scrub starts 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: audit 2023-10-09T04:52:02.535352+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: audit 2023-10-09T04:52:02.566934+0000 mon.a (mon.0) 1671 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: audit 2023-10-09T04:52:02.568432+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: audit 2023-10-09T04:52:02.573299+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:03 smithi082 bash[79733]: audit 2023-10-09T04:52:02.592465+0000 mon.a (mon.0) 1674 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:03.521 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:03.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:03.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:03.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:03.768 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:03.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:03.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:03.931 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:03.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:04.034 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:04.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:04.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:04 smithi153 bash[59565]: cluster 2023-10-09T04:52:02.612010+0000 osd.6 (osd.6) 155 : cluster 0 9.0 scrub starts 2023-10-09T04:52:04.091 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:04 smithi153 bash[59565]: cluster 2023-10-09T04:52:02.792280+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1238: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 13089/83256 objects degraded (15.721%); 7755/83256 objects misplaced (9.315%); 5.0 MiB/s, 125 objects/s recovering 2023-10-09T04:52:04.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:04.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:04.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:04 smithi082 bash[77063]: cluster 2023-10-09T04:52:02.612010+0000 osd.6 (osd.6) 155 : cluster 0 9.0 scrub starts 2023-10-09T04:52:04.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:04 smithi082 bash[77063]: cluster 2023-10-09T04:52:02.792280+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1238: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 13089/83256 objects degraded (15.721%); 7755/83256 objects misplaced (9.315%); 5.0 MiB/s, 125 objects/s recovering 2023-10-09T04:52:04.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:04 smithi082 bash[79733]: cluster 2023-10-09T04:52:02.612010+0000 osd.6 (osd.6) 155 : cluster 0 9.0 scrub starts 2023-10-09T04:52:04.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:04 smithi082 bash[79733]: cluster 2023-10-09T04:52:02.792280+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1238: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 13089/83256 objects degraded (15.721%); 7755/83256 objects misplaced (9.315%); 5.0 MiB/s, 125 objects/s recovering 2023-10-09T04:52:04.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:04.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:04.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:04.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:04.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:04.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:04.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:04.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:04.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:04.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:04.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:04.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:04.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:04.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:05.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:05.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:52:05.258 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:05.267 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:05 smithi082 bash[79733]: cluster 2023-10-09T04:52:05.073638+0000 mon.a (mon.0) 1675 : cluster 3 Health check update: Degraded data redundancy: 13089/83256 objects degraded (15.721%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:05.267 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:05 smithi082 bash[77063]: cluster 2023-10-09T04:52:05.073638+0000 mon.a (mon.0) 1675 : cluster 3 Health check update: Degraded data redundancy: 13089/83256 objects degraded (15.721%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:05.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:05.398 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:05.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:05 smithi153 bash[59565]: cluster 2023-10-09T04:52:05.073638+0000 mon.a (mon.0) 1675 : cluster 3 Health check update: Degraded data redundancy: 13089/83256 objects degraded (15.721%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:05.436 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:05.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:05.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:05.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:05.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:05.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:05.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:05.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:05.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:05.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:06.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:06.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:06.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:06.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.325 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:06 smithi082 bash[77063]: cluster 2023-10-09T04:52:04.792970+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1239: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13089/80298 objects degraded (16.301%); 7755/80298 objects misplaced (9.658%); 3.3 MiB/s, 100 objects/s recovering 2023-10-09T04:52:06.326 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:06 smithi082 bash[79733]: cluster 2023-10-09T04:52:04.792970+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1239: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13089/80298 objects degraded (16.301%); 7755/80298 objects misplaced (9.658%); 3.3 MiB/s, 100 objects/s recovering 2023-10-09T04:52:06.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:06 smithi153 bash[59565]: cluster 2023-10-09T04:52:04.792970+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1239: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13089/80298 objects degraded (16.301%); 7755/80298 objects misplaced (9.658%); 3.3 MiB/s, 100 objects/s recovering 2023-10-09T04:52:06.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:06.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:06.582 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.610 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:06.714 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:06.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:06.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:06.901 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:06.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:07.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:07.032 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:07.139 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:07.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:07.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:07.291 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:52:07.291 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126109 9.10 9:09ff7f08:::benchmark_data_smithi082_200616_object16748:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827692+0000", 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.453087043, 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3205023769999999, 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.292 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126109 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827692+0000", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827692+0000", 2023-10-09T04:52:07.293 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827691+0000", 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827698+0000", 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7540000000000006e-06 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.294 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827712+0000", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4131000000000001e-05 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827723+0000", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0093e-05 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827788+0000", 2023-10-09T04:52:07.295 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5738e-05 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827797+0000", 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9760000000000001e-06 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.148004+0000", 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.320206795 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.296 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.148195+0000", 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0001907 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126126 9.10 9:08c268e0:::benchmark_data_smithi082_200616_object16765:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827733+0000", 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.453045964, 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.332060451, 2023-10-09T04:52:07.297 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126126 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827733+0000", 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.298 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827733+0000", 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827733+0000", 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.299 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827736+0000", 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5069999999999999e-06 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827741+0000", 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.9030000000000003e-06 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827745+0000", 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.476e-06 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.300 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827827+0000", 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1749000000000006e-05 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827833+0000", 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.613e-06 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.301 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.148493+0000", 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.320659872 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.148666+0000", 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000172428 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.149008+0000", 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00034249600000000002 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.302 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.153830+0000", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.004821695 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154058+0000", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00022779000000000001 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154136+0000", 2023-10-09T04:52:07.303 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8237000000000003e-05 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154210+0000", 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4402999999999999e-05 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159384+0000", 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0051739610000000004 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.304 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159491+0000", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010702 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159572+0000", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0928999999999997e-05 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159794+0000", 2023-10-09T04:52:07.305 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000221702 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126138 9.10 9:0bb8c178:::benchmark_data_smithi082_200616_object16777:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827751+0000", 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.453028308, 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3324479390000001, 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.306 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126138 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827751+0000", 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.307 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827751+0000", 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827751+0000", 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827753+0000", 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.052e-06 2023-10-09T04:52:07.308 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827760+0000", 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5719999999999997e-06 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827765+0000", 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0810000000000004e-06 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.309 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827862+0000", 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7521000000000004e-05 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827866+0000", 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1799999999999998e-06 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.151744+0000", 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3238778010000001 2023-10-09T04:52:07.310 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.151892+0000", 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014748499999999999 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.152202+0000", 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031065500000000001 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.311 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.158951+0000", 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0067490750000000002 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159040+0000", 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8494999999999999e-05 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159127+0000", 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7457e-05 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159324+0000", 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00019659000000000001 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159867+0000", 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00054338699999999999 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.313 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159923+0000", 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5974999999999999e-05 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159998+0000", 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4573000000000006e-05 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160199+0000", 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020133 2023-10-09T04:52:07.314 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126148 9.10 9:0b34aa09:::benchmark_data_smithi082_200616_object16787:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827768+0000", 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.453011123, 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.339743731, 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.315 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126148 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827768+0000", 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.316 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827768+0000", 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827768+0000", 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827770+0000", 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.871e-06 2023-10-09T04:52:07.317 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827774+0000", 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0400000000000003e-06 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827778+0000", 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0130000000000004e-06 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.318 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827893+0000", 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000115172 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827897+0000", 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1180000000000002e-06 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154260+0000", 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3263623149999999 2023-10-09T04:52:07.319 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154478+0000", 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021831500000000001 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.154870+0000", 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000391856 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.320 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159141+0000", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0042711559999999999 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.159441+0000", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000300067 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160287+0000", 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00084571400000000001 2023-10-09T04:52:07.321 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160346+0000", 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9806999999999998e-05 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160393+0000", 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6523000000000002e-05 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.322 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167261+0000", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068680030000000001 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167330+0000", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9305000000000001e-05 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167512+0000", 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000181736 2023-10-09T04:52:07.323 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126174 9.10 9:0d1b1222:::benchmark_data_smithi082_200616_object16813:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827781+0000", 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.452998116, 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3401271370000001, 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.324 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126174 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827781+0000", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827781+0000", 2023-10-09T04:52:07.325 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827781+0000", 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827783+0000", 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.99e-06 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.326 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827787+0000", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0759999999999996e-06 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827791+0000", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9179999999999996e-06 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827922+0000", 2023-10-09T04:52:07.327 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013084899999999999 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827926+0000", 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1540000000000004e-06 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.157010+0000", 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3290836779999999 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.328 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.157236+0000", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000226331 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.157574+0000", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00033786500000000002 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160473+0000", 2023-10-09T04:52:07.329 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0028990180000000002 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160521+0000", 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7883000000000002e-05 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160647+0000", 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012575800000000001 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.330 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160700+0000", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3353999999999998e-05 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167556+0000", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0068557119999999999 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167691+0000", 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013530200000000001 2023-10-09T04:52:07.331 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167743+0000", 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2401000000000002e-05 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167908+0000", 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000165124 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.332 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126208 9.10 9:0ca34bb6:::benchmark_data_smithi082_200616_object16847:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827794+0000", 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45298558499999, 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.347471788, 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126208 2023-10-09T04:52:07.333 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827794+0000", 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827794+0000", 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.334 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827794+0000", 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827795+0000", 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9240000000000001e-06 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.335 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827800+0000", 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1969999999999998e-06 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827804+0000", 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2889999999999999e-06 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827952+0000", 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000147832 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.336 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827956+0000", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2119999999999997e-06 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160735+0000", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3327789619999999 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.160914+0000", 2023-10-09T04:52:07.337 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00017908799999999999 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.161268+0000", 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00035394499999999999 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167594+0000", 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0063259379999999997 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.338 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170094+0000", 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0024998160000000002 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170162+0000", 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8433e-05 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.339 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174714+0000", 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0045513230000000003 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174778+0000", 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4931000000000006e-05 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175120+0000", 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00034111599999999998 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.340 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175157+0000", 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7051000000000002e-05 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175266+0000", 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000109002 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.341 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126241 9.10 9:0a08d892:::benchmark_data_smithi082_200616_object16880:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827828+0000", 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45295163, 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3476923089999999, 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126241 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.342 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827828+0000", 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827828+0000", 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.343 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827827+0000", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827830+0000", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7429999999999998e-06 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827835+0000", 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0669999999999999e-06 2023-10-09T04:52:07.344 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827840+0000", 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2939999999999999e-06 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827981+0000", 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014185599999999999 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.345 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827985+0000", 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0640000000000004e-06 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.164451+0000", 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.336465539 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.164642+0000", 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000190908 2023-10-09T04:52:07.346 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.165046+0000", 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00040454 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167678+0000", 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0026316299999999998 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.347 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170129+0000", 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0024508899999999998 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170197+0000", 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.7626999999999995e-05 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174722+0000", 2023-10-09T04:52:07.348 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0045257370000000002 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174984+0000", 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00026131199999999998 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175326+0000", 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000342839 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.349 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175380+0000", 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3149999999999998e-05 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175520+0000", 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014046100000000001 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.350 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126263 9.10 9:0d7daea4:::benchmark_data_smithi082_200616_object16902:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827862+0000", 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45291736599999, 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.353545413, 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126263 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.351 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827862+0000", 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827862+0000", 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.352 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827862+0000", 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827864+0000", 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.666e-06 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827869+0000", 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6600000000000003e-06 2023-10-09T04:52:07.353 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827873+0000", 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8099999999999999e-06 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828012+0000", 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013900499999999999 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.354 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828016+0000", 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9659999999999998e-06 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.167951+0000", 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.339934961 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.168088+0000", 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000137124 2023-10-09T04:52:07.355 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.168366+0000", 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00027780699999999998 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174833+0000", 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064675749999999997 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.356 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174844+0000", 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0509999999999999e-05 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174891+0000", 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7037999999999999e-05 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174942+0000", 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1063999999999998e-05 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.357 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175058+0000", 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000116233 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181220+0000", 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0061621519999999997 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.358 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181272+0000", 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1487999999999999e-05 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181407+0000", 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000135714 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126280 9.10 9:0a3ccbfd:::benchmark_data_smithi082_200616_object16919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.359 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827893+0000", 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.452885982, 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.373320396, 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126280 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.360 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827893+0000", 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827893+0000", 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827893+0000", 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.361 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827896+0000", 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.004e-06 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827902+0000", 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6110000000000001e-06 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.362 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827906+0000", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1400000000000002e-06 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828042+0000", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013616499999999999 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828046+0000", 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0840000000000002e-06 2023-10-09T04:52:07.363 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170221+0000", 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.342174966 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170353+0000", 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013161999999999999 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.364 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.170671+0000", 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031838899999999998 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.174851+0000", 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0041799489999999996 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175028+0000", 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000177267 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.365 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175098+0000", 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9575999999999999e-05 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181188+0000", 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0060899120000000003 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.366 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181446+0000", 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00025862299999999998 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200936+0000", 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019489475999999999 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201000+0000", 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3880000000000005e-05 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.367 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201214+0000", 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000214095 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126311 9.10 9:0f8082be:::benchmark_data_smithi082_200616_object16950:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827909+0000", 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45287007899999, 2023-10-09T04:52:07.368 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.373496048, 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126311 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827909+0000", 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.369 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827909+0000", 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827909+0000", 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.370 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827911+0000", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0770000000000002e-06 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827915+0000", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.109e-06 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827919+0000", 2023-10-09T04:52:07.371 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7579999999999998e-06 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828074+0000", 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015449600000000001 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828077+0000", 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8800000000000001e-06 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.372 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.172457+0000", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.344379464 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.172608+0000", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015083100000000001 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.172936+0000", 2023-10-09T04:52:07.373 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00032807499999999999 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175039+0000", 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0021028510000000002 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181151+0000", 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0061127459999999996 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.374 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181479+0000", 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00032726100000000002 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181511+0000", 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.2613000000000001e-05 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.375 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187314+0000", 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0058027160000000003 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201278+0000", 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013964047 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201315+0000", 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6872000000000002e-05 2023-10-09T04:52:07.376 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201405+0000", 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0527999999999998e-05 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126317 9.10 9:0d103556:::benchmark_data_smithi082_200616_object16956:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827922+0000", 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.452857182, 2023-10-09T04:52:07.377 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3737626490000001, 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126317 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827922+0000", 2023-10-09T04:52:07.378 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827922+0000", 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827922+0000", 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.379 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827924+0000", 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9199999999999998e-06 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827928+0000", 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0670000000000002e-06 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.380 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827932+0000", 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9310000000000001e-06 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828105+0000", 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00017316900000000001 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828109+0000", 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9349999999999996e-06 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.381 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.175571+0000", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.347461947 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.177051+0000", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001480252 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.177408+0000", 2023-10-09T04:52:07.382 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00035696499999999999 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187364+0000", 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0099562239999999996 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187373+0000", 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7120000000000006e-06 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187504+0000", 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013122599999999999 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190254+0000", 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027497540000000001 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200533+0000", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010279183000000001 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201471+0000", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00093741799999999998 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201526+0000", 2023-10-09T04:52:07.385 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5361000000000001e-05 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201685+0000", 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015884800000000001 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.386 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126329 9.10 9:0fe4402b:::benchmark_data_smithi082_200616_object16968:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827954+0000", 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45282561, 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.374068646, 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126329 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.387 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827954+0000", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827954+0000", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827953+0000", 2023-10-09T04:52:07.388 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827959+0000", 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.4639999999999997e-06 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827964+0000", 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0309999999999998e-06 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.389 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827968+0000", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3560000000000003e-06 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828133+0000", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016515799999999999 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828137+0000", 2023-10-09T04:52:07.390 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7809999999999999e-06 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.179199+0000", 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3510614919999999 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.179320+0000", 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000121587 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.391 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.179579+0000", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000258514 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187381+0000", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0078022860000000003 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187445+0000", 2023-10-09T04:52:07.392 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.4053000000000005e-05 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.188277+0000", 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00083201300000000005 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190302+0000", 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0020246299999999999 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.393 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200583+0000", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010281261999999999 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201902+0000", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0013192099999999999 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201934+0000", 2023-10-09T04:52:07.394 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1779999999999997e-05 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202022+0000", 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8430000000000005e-05 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126343 9.10 9:0eb11693:::benchmark_data_smithi082_200616_object16982:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.827983+0000", 2023-10-09T04:52:07.395 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.452796791, 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.374302369, 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126343 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.396 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827983+0000", 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827983+0000", 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827982+0000", 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.397 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827985+0000", 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7970000000000001e-06 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827990+0000", 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7899999999999999e-06 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.398 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.827994+0000", 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9589999999999997e-06 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828163+0000", 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016922500000000001 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828167+0000", 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0890000000000002e-06 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.399 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181534+0000", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.353367225 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181662+0000", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012789299999999999 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.181927+0000", 2023-10-09T04:52:07.400 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00026503600000000002 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187391+0000", 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0054642270000000003 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190344+0000", 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0029520089999999998 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.401 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190379+0000", 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5714999999999999e-05 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190414+0000", 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4374e-05 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.402 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200625+0000", 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010211354000000001 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202077+0000", 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0014523610000000001 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202129+0000", 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1267000000000002e-05 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.403 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202285+0000", 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000156432 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126427 9.10 9:0f85e38e:::benchmark_data_smithi082_200616_object17066:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.828013+0000", 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.45276634699999, 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3744507750000001, 2023-10-09T04:52:07.404 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126427 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828013+0000", 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.405 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828013+0000", 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828013+0000", 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.406 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828015+0000", 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6299999999999998e-06 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828021+0000", 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7969999999999997e-06 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828025+0000", 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8519999999999997e-06 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.407 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828193+0000", 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000168025 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828197+0000", 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8809999999999998e-06 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.408 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.183471+0000", 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.355273776 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.183602+0000", 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013154200000000001 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.183837+0000", 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00023435400000000001 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.409 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.187399+0000", 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003562618 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190453+0000", 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0030535940000000002 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.410 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200668+0000", 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010214968 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200765+0000", 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6994e-05 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200807+0000", 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1811000000000003e-05 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.411 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202336+0000", 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001529484 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202373+0000", 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7153000000000002e-05 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.412 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202464+0000", 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0661000000000005e-05 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126432 9.10 9:0ded31b0:::benchmark_data_smithi082_200616_object17071:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.828029+0000", 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.452750111, 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.374649872, 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.413 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126432 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828029+0000", 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.414 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828029+0000", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828029+0000", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828031+0000", 2023-10-09T04:52:07.415 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9199999999999998e-06 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828035+0000", 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9489999999999998e-06 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828039+0000", 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9620000000000004e-06 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.416 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828220+0000", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018163200000000001 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828225+0000", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1459999999999998e-06 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.185471+0000", 2023-10-09T04:52:07.417 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.357246567 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.185610+0000", 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013832699999999999 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.185917+0000", 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00030740100000000001 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.418 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.190264+0000", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0043472049999999998 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200723+0000", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010458419 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200848+0000", 2023-10-09T04:52:07.419 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012550199999999999 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201729+0000", 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00088078699999999995 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201816+0000", 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7448999999999996e-05 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.420 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202561+0000", 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00074457899999999999 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202594+0000", 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.3500000000000001e-05 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.421 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202679+0000", 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.4815999999999996e-05 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:126515 9.10 9:0931abfc:::benchmark_data_smithi082_200616_object17154:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e262)", 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:50:17.828044+0000", 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 109.4527355, 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.375761091, 2023-10-09T04:52:07.422 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 126515 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828044+0000", 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.423 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828044+0000", 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828044+0000", 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.424 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828046+0000", 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.9989999999999998e-06 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828050+0000", 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.0520000000000003e-06 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828055+0000", 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2320000000000001e-06 2023-10-09T04:52:07.425 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828241+0000", 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000186181 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:17.828245+0000", 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9570000000000004e-06 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.188315+0000", 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3600702339999999 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.188470+0000", 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015424300000000001 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,2,4,6,7", 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.188780+0000", 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00031061100000000003 2023-10-09T04:52:07.427 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.200894+0000", 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.012113547000000001 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201417+0000", 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000523081 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201780+0000", 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00036336499999999999 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.201867+0000", 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6559999999999998e-05 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.202520+0000", 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00065292699999999998 2023-10-09T04:52:07.429 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.203649+0000", 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0011295260000000001 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.203699+0000", 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0076e-05 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.430 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:50:19.203805+0000", 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00010577 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162034 9.4 9:2438abc3:::benchmark_data_smithi082_200616_object52673:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e280)", 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:14.097388+0000", 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 53.183390930000002, 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.244416438, 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.431 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162034 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097388+0000", 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.432 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097388+0000", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097386+0000", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097396+0000", 2023-10-09T04:52:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.9150000000000008e-06 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097412+0000", 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.5364000000000001e-05 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097422+0000", 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0414000000000001e-05 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.434 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097556+0000", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013406 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op must wait for map", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097564+0000", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.345e-06 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101623+0000", 2023-10-09T04:52:07.435 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040593479999999999 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101632+0000", 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4299999999999995e-06 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.321070+0000", 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2194381080000001 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.321222+0000", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000151287 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.321677+0000", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00045489599999999999 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.331365+0000", 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0096886469999999999 2023-10-09T04:52:07.437 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.334827+0000", 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0034617810000000001 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.340164+0000", 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.005337138 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.438 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.340217+0000", 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2366e-05 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.341805+0000", 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0015884110000000001 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162039 9.4 9:2524beb6:::benchmark_data_smithi082_200616_object52678:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e280)", 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:14.097443+0000", 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 53.183336212999997, 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2456357140000001, 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162039 2023-10-09T04:52:07.440 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097443+0000", 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097443+0000", 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.441 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097443+0000", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097447+0000", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4070000000000003e-06 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097456+0000", 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6400000000000003e-06 2023-10-09T04:52:07.442 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097463+0000", 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9580000000000001e-06 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097620+0000", 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015709200000000001 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.443 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097626+0000", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1979999999999998e-06 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101655+0000", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040293359999999997 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101662+0000", 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8669999999999996e-06 2023-10-09T04:52:07.444 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.325146+0000", 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.223484311 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.325305+0000", 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015835799999999999 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.445 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.325740+0000", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00043531700000000001 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.331404+0000", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056634859999999997 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.334838+0000", 2023-10-09T04:52:07.446 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003434505 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.341865+0000", 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0070268179999999998 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.341990+0000", 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012557900000000001 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.447 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.343079+0000", 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0010884670000000001 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162045 9.4 9:22461f48:::benchmark_data_smithi082_200616_object52684:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e280)", 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:14.097469+0000", 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 53.183310290000001, 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2536061519999999, 2023-10-09T04:52:07.448 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162045 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097469+0000", 2023-10-09T04:52:07.449 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097469+0000", 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097469+0000", 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.450 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097472+0000", 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5310000000000002e-06 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097479+0000", 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8709999999999999e-06 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.451 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097485+0000", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.772e-06 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097649+0000", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00016458600000000001 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097654+0000", 2023-10-09T04:52:07.452 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.797e-06 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101696+0000", 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040412959999999998 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101703+0000", 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1400000000000002e-06 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.453 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.328176+0000", 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2264731170000001 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.328332+0000", 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015638800000000001 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:52:07.454 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.328860+0000", 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00052772099999999998 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.334845+0000", 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0059846609999999996 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.336226+0000", 2023-10-09T04:52:07.455 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001381703 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.349933+0000", 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.01370678 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.349986+0000", 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3188e-05 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.456 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.351075+0000", 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0010890730000000001 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162072 9.4 9:24605b76:::benchmark_data_smithi082_200616_object52711:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e280)", 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:14.097490+0000", 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 53.183289113999997, 2023-10-09T04:52:07.457 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2547902739999999, 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162072 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097490+0000", 2023-10-09T04:52:07.458 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097490+0000", 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097490+0000", 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.459 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097493+0000", 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.8629999999999999e-06 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097499+0000", 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.55e-06 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:52:07.460 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097505+0000", 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5860000000000004e-06 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097677+0000", 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000172406 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.097685+0000", 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2010000000000002e-06 2023-10-09T04:52:07.461 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101742+0000", 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040572009999999999 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.101750+0000", 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3499999999999997e-06 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:52:07.462 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.331877+0000", 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.230126464 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.332044+0000", 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000167913 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.332507+0000", 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000462399 2023-10-09T04:52:07.463 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.334978+0000", 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002470809 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.343388+0000", 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0084105299999999994 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.464 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.351121+0000", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0077324550000000001 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.351167+0000", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6663999999999999e-05 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:15.352281+0000", 2023-10-09T04:52:07.465 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0011132959999999999 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:52:07.466 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:52:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:07.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:07.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:07.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:07.698 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:07.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:07.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:07.828 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:07.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:07.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:07.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:07.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.107 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:08 smithi153 bash[59565]: cluster 2023-10-09T04:52:06.793725+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1240: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 773 op/s; 12627/73224 objects degraded (17.244%); 6763/73224 objects misplaced (9.236%); 5.1 MiB/s, 154 objects/s recovering 2023-10-09T04:52:08.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.246 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:08 smithi082 bash[77063]: cluster 2023-10-09T04:52:06.793725+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1240: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 773 op/s; 12627/73224 objects degraded (17.244%); 6763/73224 objects misplaced (9.236%); 5.1 MiB/s, 154 objects/s recovering 2023-10-09T04:52:08.246 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:08 smithi082 bash[79733]: cluster 2023-10-09T04:52:06.793725+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1240: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 773 op/s; 12627/73224 objects degraded (17.244%); 6763/73224 objects misplaced (9.236%); 5.1 MiB/s, 154 objects/s recovering 2023-10-09T04:52:08.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:08.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:08.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.566 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:08.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:08.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:08.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:09.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:09.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:09.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:09.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:09.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:09.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:09.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:09.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:09.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:09.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:09.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:09.768 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:09.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:09.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:09.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:09.996 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:09] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-10-09T04:52:10.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:10.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:10.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:10.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:10.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:10.281 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:10.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:10.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:10 smithi153 bash[59565]: cluster 2023-10-09T04:52:08.794393+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1241: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 12627/71910 objects degraded (17.559%); 6763/71910 objects misplaced (9.405%); 3.5 MiB/s, 106 objects/s recovering 2023-10-09T04:52:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:10 smithi153 bash[59565]: cluster 2023-10-09T04:52:10.074864+0000 mon.a (mon.0) 1676 : cluster 3 Health check update: Degraded data redundancy: 12627/71910 objects degraded (17.559%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:10.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:10.543 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:10.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:10.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:10.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:10.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:10.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:10.906 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:10 smithi082 bash[77063]: cluster 2023-10-09T04:52:08.794393+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1241: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 12627/71910 objects degraded (17.559%); 6763/71910 objects misplaced (9.405%); 3.5 MiB/s, 106 objects/s recovering 2023-10-09T04:52:10.906 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:10 smithi082 bash[77063]: cluster 2023-10-09T04:52:10.074864+0000 mon.a (mon.0) 1676 : cluster 3 Health check update: Degraded data redundancy: 12627/71910 objects degraded (17.559%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:10.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:10.911 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:10 smithi082 bash[79733]: cluster 2023-10-09T04:52:08.794393+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1241: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 12627/71910 objects degraded (17.559%); 6763/71910 objects misplaced (9.405%); 3.5 MiB/s, 106 objects/s recovering 2023-10-09T04:52:10.911 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:10 smithi082 bash[79733]: cluster 2023-10-09T04:52:10.074864+0000 mon.a (mon.0) 1676 : cluster 3 Health check update: Degraded data redundancy: 12627/71910 objects degraded (17.559%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:10.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:10.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:11.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:11.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:11.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:11.324 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:11.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:11.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:11.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:11.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:11.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:11.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:11.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:11.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:11.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:12.033 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:12.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:12.162 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:12.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:12.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:12 smithi082 bash[77063]: cluster 2023-10-09T04:52:10.795015+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1242: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 12627/66672 objects degraded (18.939%); 6537/66672 objects misplaced (9.805%); 3.5 MiB/s, 137 objects/s recovering 2023-10-09T04:52:12.191 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:12 smithi082 bash[79733]: cluster 2023-10-09T04:52:10.795015+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1242: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 12627/66672 objects degraded (18.939%); 6537/66672 objects misplaced (9.805%); 3.5 MiB/s, 137 objects/s recovering 2023-10-09T04:52:12.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:52:12.291 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:12.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:12.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:12 smithi153 bash[59565]: cluster 2023-10-09T04:52:10.795015+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1242: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 12627/66672 objects degraded (18.939%); 6537/66672 objects misplaced (9.805%); 3.5 MiB/s, 137 objects/s recovering 2023-10-09T04:52:12.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:12.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:12.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:12.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:12.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:12.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:12.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:12.865 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:12.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:12.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:12.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:13.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:13.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.148 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:13.251 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:13.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:13.420 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:13.524 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:13.551 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:13.654 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.691 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:13.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.818 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:13.924 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:13.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:14.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:14.139 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:52:14.149 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:52:14.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:14.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.389 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:14 smithi153 bash[59565]: cluster 2023-10-09T04:52:12.554036+0000 osd.6 (osd.6) 156 : cluster 0 9.0 scrub starts 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:14 smithi153 bash[59565]: cluster 2023-10-09T04:52:12.795729+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1243: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 12199/62199 objects degraded (19.613%); 5794/62199 objects misplaced (9.315%); 5.2 MiB/s, 140 objects/s recovering 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:14 smithi153 bash[59565]: audit 2023-10-09T04:52:13.265518+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:14 smithi153 bash[59565]: audit 2023-10-09T04:52:13.429149+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.82:0/1837088883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.391 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:14 smithi153 bash[59565]: audit 2023-10-09T04:52:13.429722+0000 mon.a (mon.0) 1678 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:14.401 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[77063]: cluster 2023-10-09T04:52:12.554036+0000 osd.6 (osd.6) 156 : cluster 0 9.0 scrub starts 2023-10-09T04:52:14.402 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[77063]: cluster 2023-10-09T04:52:12.795729+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1243: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 12199/62199 objects degraded (19.613%); 5794/62199 objects misplaced (9.315%); 5.2 MiB/s, 140 objects/s recovering 2023-10-09T04:52:14.402 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[77063]: audit 2023-10-09T04:52:13.265518+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:52:14.402 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[77063]: audit 2023-10-09T04:52:13.429149+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.82:0/1837088883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.402 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[77063]: audit 2023-10-09T04:52:13.429722+0000 mon.a (mon.0) 1678 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.403 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[79733]: cluster 2023-10-09T04:52:12.554036+0000 osd.6 (osd.6) 156 : cluster 0 9.0 scrub starts 2023-10-09T04:52:14.403 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[79733]: cluster 2023-10-09T04:52:12.795729+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1243: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 12199/62199 objects degraded (19.613%); 5794/62199 objects misplaced (9.315%); 5.2 MiB/s, 140 objects/s recovering 2023-10-09T04:52:14.403 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[79733]: audit 2023-10-09T04:52:13.265518+0000 mon.a (mon.0) 1677 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:52:14.404 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[79733]: audit 2023-10-09T04:52:13.429149+0000 mon.c (mon.2) 124 : audit 1 from='client.? 172.21.15.82:0/1837088883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.404 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:14 smithi082 bash[79733]: audit 2023-10-09T04:52:13.429722+0000 mon.a (mon.0) 1678 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:14.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.524 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:14.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:14.669 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:14.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:14.905 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:14.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:14.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:15.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:15.084 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:15.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:15.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:15.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:15.330 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:15.339 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.118006+0000 mon.a (mon.0) 1679 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.126003+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: audit 2023-10-09T04:52:14.128604+0000 mon.a (mon.0) 1680 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: audit 2023-10-09T04:52:14.128741+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.128807+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.137965+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: audit 2023-10-09T04:52:14.301949+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: audit 2023-10-09T04:52:14.305685+0000 mon.a (mon.0) 1684 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.075727+0000 mon.a (mon.0) 1685 : cluster 3 Health check update: Degraded data redundancy: 12199/62199 objects degraded (19.613%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:15.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:15 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.081989+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T04:52:15.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:15.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:15.463 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.118006+0000 mon.a (mon.0) 1679 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.126003+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: audit 2023-10-09T04:52:14.128604+0000 mon.a (mon.0) 1680 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:52:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: audit 2023-10-09T04:52:14.128741+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.128807+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.137965+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: audit 2023-10-09T04:52:14.301949+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: audit 2023-10-09T04:52:14.305685+0000 mon.a (mon.0) 1684 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.075727+0000 mon.a (mon.0) 1685 : cluster 3 Health check update: Degraded data redundancy: 12199/62199 objects degraded (19.613%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.081989+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.118006+0000 mon.a (mon.0) 1679 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.126003+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: audit 2023-10-09T04:52:14.128604+0000 mon.a (mon.0) 1680 : audit 1 from='client.? 172.21.15.82:0/3410268330' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:52:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: audit 2023-10-09T04:52:14.128741+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.128807+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.137965+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: audit 2023-10-09T04:52:14.301949+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: audit 2023-10-09T04:52:14.305685+0000 mon.a (mon.0) 1684 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.075727+0000 mon.a (mon.0) 1685 : cluster 3 Health check update: Degraded data redundancy: 12199/62199 objects degraded (19.613%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:15.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:15 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.081989+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-09T04:52:15.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:15.601 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:15.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:15.729 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:15.752 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:15.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:15.879 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:15.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:15.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:15.982 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:16.010 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:16.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:16.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:16.242 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:16.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:16.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:52:16.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:16.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.162889+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:16.411 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:14.796415+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1245: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 12199/59217 objects degraded (20.601%); 5794/59217 objects misplaced (9.784%); 4.2 MiB/s, 135 objects/s recovering 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.082169+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.086097+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.088703+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:15.095230+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[77063]: cluster 2023-10-09T04:52:16.093807+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.162889+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:14.796415+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1245: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 12199/59217 objects degraded (20.601%); 5794/59217 objects misplaced (9.784%); 4.2 MiB/s, 135 objects/s recovering 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.082169+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.412 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.086097+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.088703+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:15.095230+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.413 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:16 smithi082 bash[79733]: cluster 2023-10-09T04:52:16.093807+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.162889+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:14.796415+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1245: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 12199/59217 objects degraded (20.601%); 5794/59217 objects misplaced (9.784%); 4.2 MiB/s, 135 objects/s recovering 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.082169+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.086097+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.088703+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:15.095230+0000 osd.6 (osd.6) 158 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T04:52:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:16 smithi153 bash[59565]: cluster 2023-10-09T04:52:16.093807+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-09T04:52:16.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:16.539 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:16.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:16.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:16.711 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:16.721 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:16.815 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:16.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:16.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:16.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:16.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:17.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:17.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:17.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:17.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:17 smithi153 bash[59565]: cluster 2023-10-09T04:52:16.089703+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.251 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:17 smithi153 bash[59565]: cluster 2023-10-09T04:52:16.097259+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:17.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:17.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:17 smithi082 bash[77063]: cluster 2023-10-09T04:52:16.089703+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:17 smithi082 bash[77063]: cluster 2023-10-09T04:52:16.097259+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:17 smithi082 bash[79733]: cluster 2023-10-09T04:52:16.089703+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:17 smithi082 bash[79733]: cluster 2023-10-09T04:52:16.097259+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:17.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:17.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:17.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:17.637 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:17.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:17.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:17.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:17.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:17.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:17.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:17.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:18.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:18.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:18.150 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:18 smithi153 bash[59565]: cluster 2023-10-09T04:52:16.144691+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:18.150 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:18 smithi153 bash[59565]: cluster 2023-10-09T04:52:16.797123+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1248: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 11775/53043 objects degraded (22.199%); 4888/53043 objects misplaced (9.215%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:52:18.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:18.194 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:18.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:18 smithi082 bash[77063]: cluster 2023-10-09T04:52:16.144691+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:18.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:18 smithi082 bash[77063]: cluster 2023-10-09T04:52:16.797123+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1248: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 11775/53043 objects degraded (22.199%); 4888/53043 objects misplaced (9.215%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:52:18.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:18 smithi082 bash[79733]: cluster 2023-10-09T04:52:16.144691+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T04:52:18.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:18 smithi082 bash[79733]: cluster 2023-10-09T04:52:16.797123+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1248: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 11775/53043 objects degraded (22.199%); 4888/53043 objects misplaced (9.215%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:52:18.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.327 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:18.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:52:18.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:18.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:18.567 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.594 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:18.702 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:18.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:18.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:18.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:18.879 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:18.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:19.014 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:19.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:19.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:19.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:19.397 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:19.425 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:19.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:19.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:19.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:19.670 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:19.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:19.808 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:19.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:19.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:19.939 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:19] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T04:52:19.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:19.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:20.065 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:20.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:20.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:20.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:20 smithi082 bash[77063]: cluster 2023-10-09T04:52:18.797732+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1249: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 11775/51915 objects degraded (22.681%); 4888/51915 objects misplaced (9.415%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:20 smithi082 bash[77063]: cluster 2023-10-09T04:52:20.077238+0000 mon.a (mon.0) 1688 : cluster 3 Health check update: Degraded data redundancy: 11775/51915 objects degraded (22.681%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:20 smithi082 bash[79733]: cluster 2023-10-09T04:52:18.797732+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1249: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 11775/51915 objects degraded (22.681%); 4888/51915 objects misplaced (9.415%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:20.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:20 smithi082 bash[79733]: cluster 2023-10-09T04:52:20.077238+0000 mon.a (mon.0) 1688 : cluster 3 Health check update: Degraded data redundancy: 11775/51915 objects degraded (22.681%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:20.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:20.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:20.339 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:20.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:20 smithi153 bash[59565]: cluster 2023-10-09T04:52:18.797732+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1249: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 11775/51915 objects degraded (22.681%); 4888/51915 objects misplaced (9.415%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:20 smithi153 bash[59565]: cluster 2023-10-09T04:52:20.077238+0000 mon.a (mon.0) 1688 : cluster 3 Health check update: Degraded data redundancy: 11775/51915 objects degraded (22.681%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:20.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:20.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:20.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:20.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:20.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:20.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:20.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:20.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:20.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:20.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:21.001 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:21.034 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:21.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:21.186 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:21.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:21.316 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:21.327 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:21.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:21.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:21.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:21.571 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:21.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:21.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:21.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:21.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:21.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:21.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:21.958 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:21.988 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:21.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:21.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:22.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:22.231 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:22.288 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:22 smithi082 bash[77063]: cluster 2023-10-09T04:52:20.798873+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1250: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 974 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 11775/46638 objects degraded (25.248%); 4692/46638 objects misplaced (10.060%); 3.0 MiB/s, 98 objects/s recovering 2023-10-09T04:52:22.289 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:22 smithi082 bash[79733]: cluster 2023-10-09T04:52:20.798873+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1250: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 974 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 11775/46638 objects degraded (25.248%); 4692/46638 objects misplaced (10.060%); 3.0 MiB/s, 98 objects/s recovering 2023-10-09T04:52:22.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:22.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:22.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:22.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:22.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:22.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:22 smithi153 bash[59565]: cluster 2023-10-09T04:52:20.798873+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1250: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 974 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 11775/46638 objects degraded (25.248%); 4692/46638 objects misplaced (10.060%); 3.0 MiB/s, 98 objects/s recovering 2023-10-09T04:52:22.677 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:22.782 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:22.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:22.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:22.953 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:22.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:22.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:23.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:23.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:23.204 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:23.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:23.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:23.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:23.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:23.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:23.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:23.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:23.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:23.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:23.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:23.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:23.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:23.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:23.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:23.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:24.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:24.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:24.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:24.167 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:24 smithi153 bash[59565]: cluster 2023-10-09T04:52:22.799678+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1251: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 856 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11317/40968 objects degraded (27.624%); 3884/40968 objects misplaced (9.481%); 5.2 MiB/s, 124 objects/s recovering 2023-10-09T04:52:24.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:24.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:24.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:24.290 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:24 smithi082 bash[77063]: cluster 2023-10-09T04:52:22.799678+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1251: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 856 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11317/40968 objects degraded (27.624%); 3884/40968 objects misplaced (9.481%); 5.2 MiB/s, 124 objects/s recovering 2023-10-09T04:52:24.290 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:24 smithi082 bash[79733]: cluster 2023-10-09T04:52:22.799678+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1251: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 856 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11317/40968 objects degraded (27.624%); 3884/40968 objects misplaced (9.481%); 5.2 MiB/s, 124 objects/s recovering 2023-10-09T04:52:24.309 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:24.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:24.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:24.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:24.677 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:24.715 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:24.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.863 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:24.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:24.991 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:24.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:25.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.127 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:25.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:25.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:25.365 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:25.374 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:25 smithi082 bash[77063]: cluster 2023-10-09T04:52:25.078327+0000 mon.a (mon.0) 1689 : cluster 3 Health check update: Degraded data redundancy: 11317/40968 objects degraded (27.624%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:25.375 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:25 smithi082 bash[79733]: cluster 2023-10-09T04:52:25.078327+0000 mon.a (mon.0) 1689 : cluster 3 Health check update: Degraded data redundancy: 11317/40968 objects degraded (27.624%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:25.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:25 smithi153 bash[59565]: cluster 2023-10-09T04:52:25.078327+0000 mon.a (mon.0) 1689 : cluster 3 Health check update: Degraded data redundancy: 11317/40968 objects degraded (27.624%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:25.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:25.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:25.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.654 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:25.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:25.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:25.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:25.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:26.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:26.072 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:26.170 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:26 smithi082 bash[77063]: cluster 2023-10-09T04:52:24.800340+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1252: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 842 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11317/40293 objects degraded (28.087%); 3884/40293 objects misplaced (9.639%); 4.3 MiB/s, 102 objects/s recovering 2023-10-09T04:52:26.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:26.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:26.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:26.323 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:26 smithi153 bash[59565]: cluster 2023-10-09T04:52:24.800340+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1252: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 842 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11317/40293 objects degraded (28.087%); 3884/40293 objects misplaced (9.639%); 4.3 MiB/s, 102 objects/s recovering 2023-10-09T04:52:26.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:26.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:26.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:26.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:26 smithi082 bash[79733]: cluster 2023-10-09T04:52:24.800340+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1252: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 842 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 11317/40293 objects degraded (28.087%); 3884/40293 objects misplaced (9.639%); 4.3 MiB/s, 102 objects/s recovering 2023-10-09T04:52:26.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:26.618 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:26.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:26.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:26.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:26.853 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:26.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:26.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:26.983 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:27.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:27.107 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:27.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:27.241 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:27.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:27.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:27.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:27.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:27.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:27.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:27.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:27.941 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:27.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:28.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:28.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:28.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:28.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:28.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:28.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:28.319 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:28 smithi082 bash[77063]: cluster 2023-10-09T04:52:26.801176+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1253: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 647 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10893/30924 objects degraded (35.225%); 2843/30924 objects misplaced (9.194%); 5.8 MiB/s, 124 objects/s recovering 2023-10-09T04:52:28.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:28 smithi082 bash[79733]: cluster 2023-10-09T04:52:26.801176+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1253: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 647 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10893/30924 objects degraded (35.225%); 2843/30924 objects misplaced (9.194%); 5.8 MiB/s, 124 objects/s recovering 2023-10-09T04:52:28.343 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:28.406 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:52:28.439 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:52:28.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:28.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:28.496 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:52:28.533 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:52:28.565 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:52:28.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:28.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:28 smithi153 bash[59565]: cluster 2023-10-09T04:52:26.801176+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1253: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 647 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10893/30924 objects degraded (35.225%); 2843/30924 objects misplaced (9.194%); 5.8 MiB/s, 124 objects/s recovering 2023-10-09T04:52:28.603 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:52:28.621 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:28.643 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:52:28.687 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:52:28.693 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.693 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.693 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.693 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.694 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.694 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.694 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.694 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:52:28.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:28.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:28.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:28.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:28.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:28.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:29.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:29.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:29.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:29.172 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:29.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:29.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:29.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:29 smithi153 bash[59565]: audit 2023-10-09T04:52:28.359277+0000 mon.a (mon.0) 1690 : audit 0 from='client.? 172.21.15.82:0/1468198479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:29.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:29 smithi153 bash[59565]: cluster 2023-10-09T04:52:28.801955+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1254: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 619 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10893/29586 objects degraded (36.818%); 2843/29586 objects misplaced (9.609%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:29.440 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:29 smithi082 bash[79733]: audit 2023-10-09T04:52:28.359277+0000 mon.a (mon.0) 1690 : audit 0 from='client.? 172.21.15.82:0/1468198479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:29 smithi082 bash[79733]: cluster 2023-10-09T04:52:28.801955+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1254: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 619 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10893/29586 objects degraded (36.818%); 2843/29586 objects misplaced (9.609%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:29.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:29 smithi082 bash[77063]: audit 2023-10-09T04:52:28.359277+0000 mon.a (mon.0) 1690 : audit 0 from='client.? 172.21.15.82:0/1468198479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:29.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:29 smithi082 bash[77063]: cluster 2023-10-09T04:52:28.801955+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1254: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 619 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10893/29586 objects degraded (36.818%); 2843/29586 objects misplaced (9.609%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:29.546 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:29.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:29.671 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:29.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:29.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:29.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:29.811 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:29] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:52:29.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:29.936 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:29.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:29.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:30.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:30.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:30.199 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:30.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:30.323 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:30.332 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:30 smithi082 bash[77063]: audit 2023-10-09T04:52:29.294750+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:30.332 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:30 smithi082 bash[77063]: cluster 2023-10-09T04:52:30.079589+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 10893/29586 objects degraded (36.818%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:30.332 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:30 smithi082 bash[79733]: audit 2023-10-09T04:52:29.294750+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:30.332 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:30 smithi082 bash[79733]: cluster 2023-10-09T04:52:30.079589+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 10893/29586 objects degraded (36.818%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:30.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:30.459 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.469 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:30 smithi153 bash[59565]: audit 2023-10-09T04:52:29.294750+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:30.469 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:30 smithi153 bash[59565]: cluster 2023-10-09T04:52:30.079589+0000 mon.a (mon.0) 1692 : cluster 3 Health check update: Degraded data redundancy: 10893/29586 objects degraded (36.818%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:30.486 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:30.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.634 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:30.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:30.771 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:30.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:30.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:31.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:31.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:31.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:31.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:31.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:31.313 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:31 smithi082 bash[79733]: cluster 2023-10-09T04:52:30.802564+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1255: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 558 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10893/26652 objects degraded (40.871%); 2597/26652 objects misplaced (9.744%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:31.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:31 smithi082 bash[77063]: cluster 2023-10-09T04:52:30.802564+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1255: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 558 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10893/26652 objects degraded (40.871%); 2597/26652 objects misplaced (9.744%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:31.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:31.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:31 smithi153 bash[59565]: cluster 2023-10-09T04:52:30.802564+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1255: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 558 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10893/26652 objects degraded (40.871%); 2597/26652 objects misplaced (9.744%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T04:52:31.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:31.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:31.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:31.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:31.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:31.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:31.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:31.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:31.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:31.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:31.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:32.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:32.105 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:32.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:32.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:32.249 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:32.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:32.412 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:32.514 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:32.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:32.655 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:32.818 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.850 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:32.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:32.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:32.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:32.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:33.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:33.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:33.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:33.258 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:33.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:33.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:33.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:33.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:33.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:33.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:33.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:33.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:33.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:33.905 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:33.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:33.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:34.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.056 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:33 smithi082 bash[77063]: cluster 2023-10-09T04:52:32.803252+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1256: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 414 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 10501/19752 objects degraded (53.164%); 1782/19752 objects misplaced (9.022%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:34.056 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:33 smithi082 bash[79733]: cluster 2023-10-09T04:52:32.803252+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1256: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 414 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 10501/19752 objects degraded (53.164%); 1782/19752 objects misplaced (9.022%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:34.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:33 smithi153 bash[59565]: cluster 2023-10-09T04:52:32.803252+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1256: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 414 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 10501/19752 objects degraded (53.164%); 1782/19752 objects misplaced (9.022%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:34.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:34.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:34.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:34.318 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:34.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:34.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:34.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:34.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:34.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:34.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:34.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:34.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:35.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:35.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:35.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:35.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.176 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:35.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:35.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:35.379 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:35.379 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:35.421 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:35.456 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:35.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:52:35.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:35.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:35.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:35.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:35.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:35.960 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:35 smithi082 bash[77063]: cluster 2023-10-09T04:52:34.803865+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1257: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 402 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 10501/19158 objects degraded (54.813%); 1782/19158 objects misplaced (9.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:35.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:35 smithi082 bash[77063]: cluster 2023-10-09T04:52:35.081151+0000 mon.a (mon.0) 1693 : cluster 3 Health check update: Degraded data redundancy: 10501/19158 objects degraded (54.813%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:35 smithi082 bash[79733]: cluster 2023-10-09T04:52:34.803865+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1257: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 402 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 10501/19158 objects degraded (54.813%); 1782/19158 objects misplaced (9.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:35.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:35 smithi082 bash[79733]: cluster 2023-10-09T04:52:35.081151+0000 mon.a (mon.0) 1693 : cluster 3 Health check update: Degraded data redundancy: 10501/19158 objects degraded (54.813%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:36.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:36.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:36.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:36.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:36.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:35 smithi153 bash[59565]: cluster 2023-10-09T04:52:34.803865+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1257: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 402 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 10501/19158 objects degraded (54.813%); 1782/19158 objects misplaced (9.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:52:36.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:35 smithi153 bash[59565]: cluster 2023-10-09T04:52:35.081151+0000 mon.a (mon.0) 1693 : cluster 3 Health check update: Degraded data redundancy: 10501/19158 objects degraded (54.813%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:36.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:36.299 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:36.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:36.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:36.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:36.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:36.583 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:36.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:36.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:36.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:36.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:36.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:36.892 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:52:36.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:36.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:36 smithi082 bash[77063]: audit 2023-10-09T04:52:36.348076+0000 mon.a (mon.0) 1694 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:52:36.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:36 smithi082 bash[79733]: audit 2023-10-09T04:52:36.348076+0000 mon.a (mon.0) 1694 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:52:37.007 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:37.111 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:37.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:37.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:37.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:36 smithi153 bash[59565]: audit 2023-10-09T04:52:36.348076+0000 mon.a (mon.0) 1694 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:52:37.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:37.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:37.380 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:37.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:37.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:37.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:37.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:37.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:37.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:37.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:37.901 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.804715+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1258: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 203 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 10057/9615 objects degraded (104.597%); 772/9615 objects misplaced (8.029%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.879117+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.879318+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: audit 2023-10-09T04:52:36.881699+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.881768+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.885802+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: cluster 2023-10-09T04:52:36.904735+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[77063]: audit 2023-10-09T04:52:37.639308+0000 mon.a (mon.0) 1697 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:37.928 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.804715+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1258: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 203 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 10057/9615 objects degraded (104.597%); 772/9615 objects misplaced (8.029%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.879117+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.879318+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: audit 2023-10-09T04:52:36.881699+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.881768+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.885802+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: cluster 2023-10-09T04:52:36.904735+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:37.929 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:37 smithi082 bash[79733]: audit 2023-10-09T04:52:37.639308+0000 mon.a (mon.0) 1697 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:37.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:37.958 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:37.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:37.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:38.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:38.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.804715+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1258: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 203 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 10057/9615 objects degraded (104.597%); 772/9615 objects misplaced (8.029%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.879117+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.879318+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: audit 2023-10-09T04:52:36.881699+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.82:0/2055079075' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.881768+0000 mon.a (mon.0) 1696 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-09T04:52:38.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.885802+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: cluster 2023-10-09T04:52:36.904735+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T04:52:38.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:37 smithi153 bash[59565]: audit 2023-10-09T04:52:37.639308+0000 mon.a (mon.0) 1697 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:38.199 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:38.239 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:38.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:38.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:38.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:38.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:38.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:52:38.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:38.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:38.746 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:38.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:38.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:38.911 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:38.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:39.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:39.024 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[77063]: cluster 2023-10-09T04:52:37.877732+0000 mon.a (mon.0) 1698 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[77063]: cluster 2023-10-09T04:52:37.892343+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[77063]: cluster 2023-10-09T04:52:37.893988+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[77063]: audit 2023-10-09T04:52:37.894855+0000 mon.a (mon.0) 1699 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[77063]: cluster 2023-10-09T04:52:37.894903+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[79733]: cluster 2023-10-09T04:52:37.877732+0000 mon.a (mon.0) 1698 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[79733]: cluster 2023-10-09T04:52:37.892343+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[79733]: cluster 2023-10-09T04:52:37.893988+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[79733]: audit 2023-10-09T04:52:37.894855+0000 mon.a (mon.0) 1699 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:39.025 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:38 smithi082 bash[79733]: cluster 2023-10-09T04:52:37.894903+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T04:52:39.038 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:39.145 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:39.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:38 smithi153 bash[59565]: cluster 2023-10-09T04:52:37.877732+0000 mon.a (mon.0) 1698 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:39.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:38 smithi153 bash[59565]: cluster 2023-10-09T04:52:37.892343+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:38 smithi153 bash[59565]: cluster 2023-10-09T04:52:37.893988+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T04:52:39.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:38 smithi153 bash[59565]: audit 2023-10-09T04:52:37.894855+0000 mon.a (mon.0) 1699 : audit 1 from='client.? 172.21.15.82:0/367741471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:39.155 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:38 smithi153 bash[59565]: cluster 2023-10-09T04:52:37.894903+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-09T04:52:39.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:39.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:39.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:39.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:39.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:39.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:39.569 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:39.675 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:39.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:39.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:39.826 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:39] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T04:52:39.844 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:39.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:39.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:39.956 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:39 smithi153 bash[59565]: cluster 2023-10-09T04:52:38.805324+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 175 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 10057/8250 objects degraded (121.903%); 772/8250 objects misplaced (9.358%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:39.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:39.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:40.012 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 109358 objects 2023-10-09T04:52:40.012 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :162.789 2023-10-09T04:52:40.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:40.103 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:40.132 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:39 smithi082 bash[77063]: cluster 2023-10-09T04:52:38.805324+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 175 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 10057/8250 objects degraded (121.903%); 772/8250 objects misplaced (9.358%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:40.132 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:39 smithi082 bash[79733]: cluster 2023-10-09T04:52:38.805324+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 175 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 10057/8250 objects degraded (121.903%); 772/8250 objects misplaced (9.358%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:52:40.133 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:52:40.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:40.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:40.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:40.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:40.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:40.501 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:40.605 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:40.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:40.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:40.772 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:40.875 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:40.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:40.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:40.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:41.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:41.021 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:40 smithi153 bash[59565]: cluster 2023-10-09T04:52:40.082315+0000 mon.a (mon.0) 1701 : cluster 3 Health check update: Degraded data redundancy: 10057/8250 objects degraded (121.903%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:41.035 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:41.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:41.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:41.169 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:40 smithi082 bash[77063]: cluster 2023-10-09T04:52:40.082315+0000 mon.a (mon.0) 1701 : cluster 3 Health check update: Degraded data redundancy: 10057/8250 objects degraded (121.903%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:41.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:40 smithi082 bash[79733]: cluster 2023-10-09T04:52:40.082315+0000 mon.a (mon.0) 1701 : cluster 3 Health check update: Degraded data redundancy: 10057/8250 objects degraded (121.903%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:41.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:41.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:41.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:41.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:41.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:41.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:41.664 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:41.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:41.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:41.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:41.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:41.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:41.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:41.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:42.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:42.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:42.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:52:42.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:42.206 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:42 smithi153 bash[59565]: cluster 2023-10-09T04:52:40.806334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 116 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 10057/5436 objects degraded (185.007%); 600/5436 objects misplaced (11.038%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:52:42.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:42.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:42.353 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:42.355 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:42 smithi082 bash[77063]: cluster 2023-10-09T04:52:40.806334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 116 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 10057/5436 objects degraded (185.007%); 600/5436 objects misplaced (11.038%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:52:42.355 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:42 smithi082 bash[79733]: cluster 2023-10-09T04:52:40.806334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 116 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 10057/5436 objects degraded (185.007%); 600/5436 objects misplaced (11.038%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:52:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:42.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:42.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:42.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:42.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:42.667 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:42.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:42.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:42.855 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:42.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:42.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:42.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:43.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:43.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:43.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:43.249 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.278 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:43.382 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:43.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:43.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:43.563 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T04:52:43.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T04:52:43.641 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:43.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:43.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:43.798 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:43.902 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:43.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:44.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:44.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:44.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:44.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:44.344 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:44 smithi082 bash[77063]: cluster 2023-10-09T04:52:42.807044+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1263: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 27 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 9735/1161 objects degraded (838.501%); 38/1161 objects misplaced (3.273%); 4.5 MiB/s, 76 objects/s recovering 2023-10-09T04:52:44.344 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:44 smithi082 bash[79733]: cluster 2023-10-09T04:52:42.807044+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1263: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 27 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 9735/1161 objects degraded (838.501%); 38/1161 objects misplaced (3.273%); 4.5 MiB/s, 76 objects/s recovering 2023-10-09T04:52:44.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:44.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:44 smithi153 bash[59565]: cluster 2023-10-09T04:52:42.807044+0000 mgr.y (mgr.34107) 1161 : cluster 0 pgmap v1263: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 27 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 9735/1161 objects degraded (838.501%); 38/1161 objects misplaced (3.273%); 4.5 MiB/s, 76 objects/s recovering 2023-10-09T04:52:44.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:44.599 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:44.629 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:44.695 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:44.695 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:44.733 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:44.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:44.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:44.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:44.993 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.026 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:45.130 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:45.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:45.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.267 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:45 smithi153 bash[59565]: audit 2023-10-09T04:52:44.295539+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:45.267 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:45 smithi153 bash[59565]: cluster 2023-10-09T04:52:45.084415+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 9735/1161 objects degraded (838.501%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:45.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:45.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:45.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:45 smithi082 bash[77063]: audit 2023-10-09T04:52:44.295539+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:45.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:45 smithi082 bash[77063]: cluster 2023-10-09T04:52:45.084415+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 9735/1161 objects degraded (838.501%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:45.396 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:45 smithi082 bash[79733]: audit 2023-10-09T04:52:44.295539+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:45.397 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:45 smithi082 bash[79733]: cluster 2023-10-09T04:52:45.084415+0000 mon.a (mon.0) 1703 : cluster 3 Health check update: Degraded data redundancy: 9735/1161 objects degraded (838.501%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:45.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:45.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:45.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:45.645 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.673 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:45.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:45.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.937 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:45.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:46.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.063 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:46.166 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:46.194 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:46.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:46.305 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:46 smithi082 bash[77063]: cluster 2023-10-09T04:52:44.807902+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1264: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 25 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 9735/1041 objects degraded (935.159%); 38/1041 objects misplaced (3.650%); 1.8 MiB/s, 33 objects/s recovering 2023-10-09T04:52:46.305 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:46 smithi082 bash[79733]: cluster 2023-10-09T04:52:44.807902+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1264: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 25 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 9735/1041 objects degraded (935.159%); 38/1041 objects misplaced (3.650%); 1.8 MiB/s, 33 objects/s recovering 2023-10-09T04:52:46.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:46.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:46 smithi153 bash[59565]: cluster 2023-10-09T04:52:44.807902+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1264: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 25 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 9735/1041 objects degraded (935.159%); 38/1041 objects misplaced (3.650%); 1.8 MiB/s, 33 objects/s recovering 2023-10-09T04:52:46.458 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:46.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:46.686 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:46.711 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:46.724 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:46.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:46.845 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:46.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:46.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:47.073 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:47.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:47.211 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:47.238 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:47.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:47.367 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:47.470 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:47.497 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:47.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:47.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:47.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:47.753 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:47.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:47.856 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:47.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:47.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:47.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:48.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:48.143 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:48.247 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:48.277 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:48.381 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:48.393 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:48 smithi153 bash[59565]: cluster 2023-10-09T04:52:46.808755+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1265: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.5 MiB/s, 38 objects/s recovering 2023-10-09T04:52:48.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:48.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:48 smithi082 bash[77063]: cluster 2023-10-09T04:52:46.808755+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1265: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.5 MiB/s, 38 objects/s recovering 2023-10-09T04:52:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:48 smithi082 bash[79733]: cluster 2023-10-09T04:52:46.808755+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1265: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.5 MiB/s, 38 objects/s recovering 2023-10-09T04:52:48.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:48.645 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:48.754 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:48.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:48.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:48.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:48.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:48.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:49.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:49.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:49.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:49.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:49.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:49.413 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:49.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:49.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:49.678 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:49.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:49.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:49.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:49.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:49.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:49.951 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:49.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:49.962 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:49] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-09T04:52:50.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:50.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:50.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:50.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:50.213 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:50.311 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:50 smithi082 bash[77063]: cluster 2023-10-09T04:52:48.809347+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1266: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.4 MiB/s, 35 objects/s recovering 2023-10-09T04:52:50.312 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:50 smithi082 bash[77063]: cluster 2023-10-09T04:52:50.085808+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 9735/1008 objects degraded (965.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:50.312 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:50 smithi082 bash[79733]: cluster 2023-10-09T04:52:48.809347+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1266: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.4 MiB/s, 35 objects/s recovering 2023-10-09T04:52:50.312 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:50 smithi082 bash[79733]: cluster 2023-10-09T04:52:50.085808+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 9735/1008 objects degraded (965.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:50.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:50 smithi153 bash[59565]: cluster 2023-10-09T04:52:48.809347+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1266: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.4 MiB/s, 35 objects/s recovering 2023-10-09T04:52:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:50 smithi153 bash[59565]: cluster 2023-10-09T04:52:50.085808+0000 mon.a (mon.0) 1704 : cluster 3 Health check update: Degraded data redundancy: 9735/1008 objects degraded (965.774%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:52:50.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:50.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:50.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:50.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:50.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:50.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:50.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:50.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:50.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:51.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:51.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:51.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:51.326 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:52:51.335 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:51.346 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:52:51.361 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:51.367 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:52:51.388 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:52:51.414 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:52:51.443 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:52:51.463 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:51.466 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:52:51.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:51.488 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:52:51.489 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.489 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.489 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.490 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.490 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.490 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.490 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.490 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:52:51.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.604 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:51.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:51.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:51.842 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:51.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:51.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:51.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:52.086 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:52.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:52.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:52.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:52.349 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:52.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:52 smithi153 bash[59565]: cluster 2023-10-09T04:52:50.810307+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1267: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 32 objects/s recovering 2023-10-09T04:52:52.361 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:52 smithi153 bash[59565]: audit 2023-10-09T04:52:51.303514+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.82:0/1714368972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:52.372 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:52.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:52.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:52 smithi082 bash[77063]: cluster 2023-10-09T04:52:50.810307+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1267: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 32 objects/s recovering 2023-10-09T04:52:52.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:52 smithi082 bash[77063]: audit 2023-10-09T04:52:51.303514+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.82:0/1714368972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:52.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:52 smithi082 bash[79733]: cluster 2023-10-09T04:52:50.810307+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1267: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 225 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 32 objects/s recovering 2023-10-09T04:52:52.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:52 smithi082 bash[79733]: audit 2023-10-09T04:52:51.303514+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.82:0/1714368972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:52.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:52.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:52.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:52.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:52.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:52.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:52.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:52.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:52.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:53.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:53.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:53.151 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:53.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:53.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:53.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:53.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:53.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:53.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:53.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:53.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:53.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:53.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:53.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:53.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:53.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:53.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:53.939 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:53.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:54.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:54.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:54.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:54.203 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:54.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:54 smithi153 bash[59565]: cluster 2023-10-09T04:52:52.811463+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1268: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 41 objects/s recovering 2023-10-09T04:52:54.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:54.471 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:54 smithi082 bash[77063]: cluster 2023-10-09T04:52:52.811463+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1268: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 41 objects/s recovering 2023-10-09T04:52:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:54 smithi082 bash[79733]: cluster 2023-10-09T04:52:52.811463+0000 mgr.y (mgr.34107) 1166 : cluster 0 pgmap v1268: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 1.2 MiB/s, 41 objects/s recovering 2023-10-09T04:52:54.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:54.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:54.706 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:54.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:54.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:54.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:55.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:55.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:55.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:55.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:55.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:55.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:55.391 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:55.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:55.494 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:55.520 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:55.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:55.651 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:55.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:55.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:55.785 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:55.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:55.918 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:55.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:56.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:56.050 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:56.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.174 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:56.277 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:56.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:56.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:56.421 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:56 smithi153 bash[59565]: cluster 2023-10-09T04:52:54.812461+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1269: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:52:56.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:56.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:56 smithi082 bash[77063]: cluster 2023-10-09T04:52:54.812461+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1269: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:52:56.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:56 smithi082 bash[79733]: cluster 2023-10-09T04:52:54.812461+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1269: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:52:56.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:56.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.693 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:56.796 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.819 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:56.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:56.952 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:56.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:56.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:57.056 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:57.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:57.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:57.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:57.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:57.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:57.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:57.590 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:57.615 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:57.720 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:57.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:57.885 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:57.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:57.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:58.018 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:58.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.147 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:58.250 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:58.262 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:52:58 smithi153 bash[59565]: cluster 2023-10-09T04:52:56.813232+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1270: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:52:58.278 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:58.371 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:52:58 smithi082 bash[79733]: cluster 2023-10-09T04:52:56.813232+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1270: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:52:58.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:52:58 smithi082 bash[77063]: cluster 2023-10-09T04:52:56.813232+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1270: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:52:58.382 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:58.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:58.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:52:58.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:58.664 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:58.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:58.793 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:58.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:58.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:58.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:59.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.048 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:59.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:59.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:59.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:59.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:59.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:59.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:59.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:59.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:59.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:59.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:59.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:59.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:59.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:52:59.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:00.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:00.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:00.065 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:52:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:52:59] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:53:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:00.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:00.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:00.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:00.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.330 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:00 smithi153 bash[59565]: cluster 2023-10-09T04:52:58.814227+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1271: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:00 smithi153 bash[59565]: audit 2023-10-09T04:52:59.295476+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:00.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:00 smithi082 bash[77063]: cluster 2023-10-09T04:52:58.814227+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1271: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:00 smithi082 bash[77063]: audit 2023-10-09T04:52:59.295476+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:00 smithi082 bash[79733]: cluster 2023-10-09T04:52:58.814227+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1271: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:00 smithi082 bash[79733]: audit 2023-10-09T04:52:59.295476+0000 mon.a (mon.0) 1705 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:00.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:00.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:00.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:00.834 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.863 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:00.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:00.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.997 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:01.024 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:53:01.025 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:53:01.025 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:53:01.025 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:53:01.025 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:53:01.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:01.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:01.143 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:53:01.223 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:01.234 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:01 smithi153 bash[59565]: audit 2023-10-09T04:53:01.061267+0000 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:53:01.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:01.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:01.363 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:01 smithi082 bash[77063]: audit 2023-10-09T04:53:01.061267+0000 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:53:01.363 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:01 smithi082 bash[79733]: audit 2023-10-09T04:53:01.061267+0000 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:53:01.374 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:01.477 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:01.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:01.605 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:01.631 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:01.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:01.735 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:01.760 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:01.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:01.863 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:01.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:01.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:01.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:02.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:02.129 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:02.148 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:53:02.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:02.263 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:02.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:02.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:02.394 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: cluster 2023-10-09T04:53:00.815362+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1272: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: cluster 2023-10-09T04:53:01.132933+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: cluster 2023-10-09T04:53:01.135101+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: audit 2023-10-09T04:53:01.138006+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: cluster 2023-10-09T04:53:01.138072+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: audit 2023-10-09T04:53:01.449463+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.82:0/4025169345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[79733]: audit 2023-10-09T04:53:01.449873+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: cluster 2023-10-09T04:53:00.815362+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1272: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:02.395 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: cluster 2023-10-09T04:53:01.132933+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: cluster 2023-10-09T04:53:01.135101+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: audit 2023-10-09T04:53:01.138006+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:53:02.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: cluster 2023-10-09T04:53:01.138072+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T04:53:02.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: audit 2023-10-09T04:53:01.449463+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.82:0/4025169345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.396 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:02 smithi082 bash[77063]: audit 2023-10-09T04:53:01.449873+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: cluster 2023-10-09T04:53:00.815362+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1272: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 19 objects/s recovering 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: cluster 2023-10-09T04:53:01.132933+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: cluster 2023-10-09T04:53:01.135101+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: audit 2023-10-09T04:53:01.138006+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.82:0/2921983174' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: cluster 2023-10-09T04:53:01.138072+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-09T04:53:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: audit 2023-10-09T04:53:01.449463+0000 mon.c (mon.2) 126 : audit 1 from='client.? 172.21.15.82:0/4025169345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:02 smithi153 bash[59565]: audit 2023-10-09T04:53:01.449873+0000 mon.a (mon.0) 1709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:02.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:02.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:02.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:02.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:02.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:02.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:02.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:02.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:02.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:03.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:03.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:03.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:03.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:03.172 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_4' removed 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.132471+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.140836+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.141014+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: audit 2023-10-09T04:53:02.143371+0000 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.143371+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.143489+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T04:53:03.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: audit 2023-10-09T04:53:02.233145+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.82:0/3423577552' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: audit 2023-10-09T04:53:02.233636+0000 mon.a (mon.0) 1713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:03 smithi153 bash[59565]: audit 2023-10-09T04:53:02.656412+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:03.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:03.294 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:03.321 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:03.426 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:03.454 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:03.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.132471+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:03.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.140836+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.141014+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: audit 2023-10-09T04:53:02.143371+0000 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.143371+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.143489+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: audit 2023-10-09T04:53:02.233145+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.82:0/3423577552' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: audit 2023-10-09T04:53:02.233636+0000 mon.a (mon.0) 1713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[77063]: audit 2023-10-09T04:53:02.656412+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.132471+0000 mon.a (mon.0) 1710 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.140836+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.141014+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: audit 2023-10-09T04:53:02.143371+0000 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.143371+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.143489+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: audit 2023-10-09T04:53:02.233145+0000 mon.c (mon.2) 127 : audit 1 from='client.? 172.21.15.82:0/3423577552' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: audit 2023-10-09T04:53:02.233636+0000 mon.a (mon.0) 1713 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:03.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:03 smithi082 bash[79733]: audit 2023-10-09T04:53:02.656412+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:03.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:03.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:03.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:03.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:03.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:53:03.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:03.983 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:03.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:04.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:04.107 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:04.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:04.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:04.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:04.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: cluster 2023-10-09T04:53:02.816021+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1275: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:53:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: cluster 2023-10-09T04:53:03.146984+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: audit 2023-10-09T04:53:03.147789+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: cluster 2023-10-09T04:53:03.147864+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T04:53:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: cluster 2023-10-09T04:53:03.547381+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:04 smithi153 bash[59565]: cluster 2023-10-09T04:53:03.547485+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:04.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: cluster 2023-10-09T04:53:02.816021+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1275: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:53:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: cluster 2023-10-09T04:53:03.146984+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: audit 2023-10-09T04:53:03.147789+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: cluster 2023-10-09T04:53:03.147864+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: cluster 2023-10-09T04:53:03.547381+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[77063]: cluster 2023-10-09T04:53:03.547485+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: cluster 2023-10-09T04:53:02.816021+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1275: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 24 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 9735/1008 objects degraded (965.774%); 38/1008 objects misplaced (3.770%); 0 B/s, 29 objects/s recovering 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: cluster 2023-10-09T04:53:03.146984+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: audit 2023-10-09T04:53:03.147789+0000 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: cluster 2023-10-09T04:53:03.147864+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-09T04:53:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: cluster 2023-10-09T04:53:03.547381+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:04.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:04 smithi082 bash[79733]: cluster 2023-10-09T04:53:03.547485+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T04:53:04.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:04.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:04.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:04.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:04.775 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:04.879 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:04.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:04.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:05.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:05.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:05.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[79733]: cluster 2023-10-09T04:53:03.547508+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:05.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[79733]: cluster 2023-10-09T04:53:04.162048+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T04:53:05.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[79733]: cluster 2023-10-09T04:53:04.163332+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[79733]: cluster 2023-10-09T04:53:04.164302+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.177 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[79733]: cluster 2023-10-09T04:53:04.165664+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[77063]: cluster 2023-10-09T04:53:03.547508+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:05.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[77063]: cluster 2023-10-09T04:53:04.162048+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T04:53:05.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[77063]: cluster 2023-10-09T04:53:04.163332+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[77063]: cluster 2023-10-09T04:53:04.164302+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:05 smithi082 bash[77063]: cluster 2023-10-09T04:53:04.165664+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:05.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:53:05.195 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:53:05.196 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:53:05.196 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T04:53:05.196 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T04:53:05.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:05.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:05.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:05.423 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:05 smithi153 bash[59565]: cluster 2023-10-09T04:53:03.547508+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T04:53:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:05 smithi153 bash[59565]: cluster 2023-10-09T04:53:04.162048+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-09T04:53:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:05 smithi153 bash[59565]: cluster 2023-10-09T04:53:04.163332+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:05 smithi153 bash[59565]: cluster 2023-10-09T04:53:04.164302+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:05 smithi153 bash[59565]: cluster 2023-10-09T04:53:04.165664+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T04:53:05.457 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:05.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:05.657 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:05.690 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:05.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:05.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:05.843 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:05.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:05.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:05.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:05.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:05.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:06.076 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:06.105 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:06.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:06.216 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[77063]: cluster 2023-10-09T04:53:04.816492+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1278: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 11/84 objects degraded (13.095%) 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[77063]: cluster 2023-10-09T04:53:05.159831+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[77063]: cluster 2023-10-09T04:53:05.171283+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[77063]: cluster 2023-10-09T04:53:05.173550+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[77063]: cluster 2023-10-09T04:53:05.175864+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[79733]: cluster 2023-10-09T04:53:04.816492+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1278: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 11/84 objects degraded (13.095%) 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[79733]: cluster 2023-10-09T04:53:05.159831+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[79733]: cluster 2023-10-09T04:53:05.171283+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.217 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[79733]: cluster 2023-10-09T04:53:05.173550+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.218 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:06 smithi082 bash[79733]: cluster 2023-10-09T04:53:05.175864+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:53:06.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:06.338 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:06.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:06 smithi153 bash[59565]: cluster 2023-10-09T04:53:04.816492+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1278: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 11/84 objects degraded (13.095%) 2023-10-09T04:53:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:06 smithi153 bash[59565]: cluster 2023-10-09T04:53:05.159831+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:53:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:06 smithi153 bash[59565]: cluster 2023-10-09T04:53:05.171283+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:06 smithi153 bash[59565]: cluster 2023-10-09T04:53:05.173550+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T04:53:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:06 smithi153 bash[59565]: cluster 2023-10-09T04:53:05.175864+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:53:06.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:06.493 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:06.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:06.623 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:06.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:06.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:06.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:06.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:06.886 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:06.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:06.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:07.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:07.124 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:07.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:07.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:53:07.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:07.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:07.387 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:07.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:07.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:07 smithi153 bash[59565]: cluster 2023-10-09T04:53:06.817388+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1280: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:07.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:07 smithi153 bash[59565]: cluster 2023-10-09T04:53:07.169973+0000 mon.a (mon.0) 1720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:53:07.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:07 smithi082 bash[77063]: cluster 2023-10-09T04:53:06.817388+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1280: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:07.462 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:07 smithi082 bash[77063]: cluster 2023-10-09T04:53:07.169973+0000 mon.a (mon.0) 1720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:53:07.462 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:07 smithi082 bash[79733]: cluster 2023-10-09T04:53:06.817388+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1280: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:07.462 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:07 smithi082 bash[79733]: cluster 2023-10-09T04:53:07.169973+0000 mon.a (mon.0) 1720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/84 objects degraded (13.095%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:53:07.517 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:07.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:07.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:07.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:07.797 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:07.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:07.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:07.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:07.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:08.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:08.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:08.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.222 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:08.325 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:08.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:08.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:08.513 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:08.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:08.749 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:08.774 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:08.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:08.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:09.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:09.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:09.152 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:09.286 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.316 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:09.420 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.450 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:09.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:09.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:09.685 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.714 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:09.714 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:09.720 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:09.720 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:09.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:09.835 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:09] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T04:53:09.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:09.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:09.912 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:53:09.932 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:53:09.954 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:53:09.974 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:53:09.985 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:09.996 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:53:09.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:09 smithi153 bash[59565]: cluster 2023-10-09T04:53:08.818139+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1281: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:09.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:09 smithi153 bash[59565]: audit 2023-10-09T04:53:08.896747+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:09.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:09 smithi153 bash[59565]: audit 2023-10-09T04:53:08.903502+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:09.998 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:09 smithi153 bash[59565]: audit 2023-10-09T04:53:09.888452+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.82:0/1396959366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:10.017 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:10.020 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:53:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:10.042 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:53:10.065 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:53:10.070 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.070 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.070 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.070 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.071 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.071 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.071 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.071 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:53:10.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:10.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:10.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:10.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[79733]: cluster 2023-10-09T04:53:08.818139+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1281: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:10.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[79733]: audit 2023-10-09T04:53:08.896747+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:10.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[79733]: audit 2023-10-09T04:53:08.903502+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:10.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[79733]: audit 2023-10-09T04:53:09.888452+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.82:0/1396959366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[77063]: cluster 2023-10-09T04:53:08.818139+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1281: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[77063]: audit 2023-10-09T04:53:08.896747+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[77063]: audit 2023-10-09T04:53:08.903502+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:10.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:09 smithi082 bash[77063]: audit 2023-10-09T04:53:09.888452+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.82:0/1396959366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:10.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:10.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:10.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:10.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:10.417 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:10.521 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:10.547 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:10.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:10.685 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:10.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:10.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:10.919 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:10.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:10.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:11.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:11.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:11.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:11.199 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:11.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:11.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:11.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:11.454 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:11.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:11.557 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:11.694 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:11.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:11.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:11.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.973 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: audit 2023-10-09T04:53:10.755618+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:11.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: audit 2023-10-09T04:53:10.765691+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:11.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: cluster 2023-10-09T04:53:10.818917+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1282: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:11.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: audit 2023-10-09T04:53:11.186066+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:11.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: audit 2023-10-09T04:53:11.186941+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:11.974 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:11 smithi153 bash[59565]: audit 2023-10-09T04:53:11.195122+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:11.992 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:11.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:12.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:12.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: audit 2023-10-09T04:53:10.755618+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: audit 2023-10-09T04:53:10.765691+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: cluster 2023-10-09T04:53:10.818917+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1282: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: audit 2023-10-09T04:53:11.186066+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: audit 2023-10-09T04:53:11.186941+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[79733]: audit 2023-10-09T04:53:11.195122+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: audit 2023-10-09T04:53:10.755618+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: audit 2023-10-09T04:53:10.765691+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: cluster 2023-10-09T04:53:10.818917+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1282: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:12.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: audit 2023-10-09T04:53:11.186066+0000 mon.a (mon.0) 1725 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:12.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: audit 2023-10-09T04:53:11.186941+0000 mon.a (mon.0) 1726 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:12.233 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:11 smithi082 bash[77063]: audit 2023-10-09T04:53:11.195122+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:12.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:12.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:12.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:12.503 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:12.539 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:12.643 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:12.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:12.788 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:12.811 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:12.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:12.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:12.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:13.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:13.071 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:13.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:13.201 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:13.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:13.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:13.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:13.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:13.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:13.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:13.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:13.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:13.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:13.832 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:13.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:13.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:13.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:13.997 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:14.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:14.113 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:13 smithi153 bash[59565]: cluster 2023-10-09T04:53:12.819810+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1283: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:14.133 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:14.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:13 smithi082 bash[77063]: cluster 2023-10-09T04:53:12.819810+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1283: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:14.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:13 smithi082 bash[79733]: cluster 2023-10-09T04:53:12.819810+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1283: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:14.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:14.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:14.319 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:53:14.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:14.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:14.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:14.533 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:14.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:14.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:14.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:14.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:14.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:14.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:14.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:14.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:15.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:15.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:15.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:15.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:15.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:15.441 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:15.471 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:15.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:15.574 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:15 smithi153 bash[59565]: audit 2023-10-09T04:53:14.300402+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:15 smithi153 bash[59565]: audit 2023-10-09T04:53:14.302293+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:15.586 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:15 smithi153 bash[59565]: cluster 2023-10-09T04:53:14.820414+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1284: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:15.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:15 smithi153 bash[59565]: audit 2023-10-09T04:53:14.999973+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:15.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[77063]: audit 2023-10-09T04:53:14.300402+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[77063]: audit 2023-10-09T04:53:14.302293+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[77063]: cluster 2023-10-09T04:53:14.820414+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1284: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[77063]: audit 2023-10-09T04:53:14.999973+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[79733]: audit 2023-10-09T04:53:14.300402+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[79733]: audit 2023-10-09T04:53:14.302293+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:15.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[79733]: cluster 2023-10-09T04:53:14.820414+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1284: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T04:53:15.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:15 smithi082 bash[79733]: audit 2023-10-09T04:53:14.999973+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:15.741 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:15.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.877 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:15.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:15.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.017 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:16.121 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.144 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:16.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:16.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:16.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:16.506 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:16.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:16.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:16.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.805 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:16.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:16.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:16.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:16.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:17.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:17.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:17.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:17.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:17.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:17.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:17.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:17.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:17.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:17.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:17.697 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:17.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:17.837 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:17.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:17.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:17.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:17.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:17 smithi082 bash[77063]: cluster 2023-10-09T04:53:16.821191+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1285: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:53:17.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:17 smithi082 bash[79733]: cluster 2023-10-09T04:53:16.821191+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1285: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:53:17.987 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:17 smithi153 bash[59565]: cluster 2023-10-09T04:53:16.821191+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1285: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:53:18.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:18.110 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:18.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:18.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:18.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:18.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:18.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:18.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:18.536 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:18.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:18.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:18.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:18.811 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:18.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:18.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:18.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:18.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:19.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:19.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:19.175 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:19.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:19.303 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.333 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:19.437 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:19.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:19.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:19.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:19.842 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:19.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:19.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:19.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:19.975 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:20.115 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:20.136 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:20.141 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:19] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T04:53:20.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:19 smithi153 bash[59565]: cluster 2023-10-09T04:53:18.821757+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1286: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:20.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:20.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:20.372 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:20.398 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:20 smithi082 bash[79733]: cluster 2023-10-09T04:53:18.821757+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1286: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:20.399 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:20 smithi082 bash[77063]: cluster 2023-10-09T04:53:18.821757+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1286: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:20.413 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:20.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.551 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:20.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:20.691 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:20.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:20.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:20.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.973 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:20.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:21.079 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:21.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:21.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:21.248 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:21.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:21.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:21.481 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:21.509 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:21.612 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:21.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:21.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:21.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:21.882 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:21.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:21.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:22.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:22.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:22.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:22.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:22.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:22.289 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:22 smithi082 bash[79733]: cluster 2023-10-09T04:53:20.822317+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1287: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:22.289 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:22 smithi082 bash[77063]: cluster 2023-10-09T04:53:20.822317+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1287: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:22.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:22.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:22.419 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:22 smithi153 bash[59565]: cluster 2023-10-09T04:53:20.822317+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1287: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:22.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:22.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:22.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:22.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:22.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:22.821 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:22.925 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:22.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:23.052 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:23.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:23.124 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_5' created 2023-10-09T04:53:23.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:23 smithi082 bash[77063]: audit 2023-10-09T04:53:22.568129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.82:0/1568097096' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.191 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:23 smithi082 bash[77063]: audit 2023-10-09T04:53:22.568497+0000 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:23 smithi082 bash[79733]: audit 2023-10-09T04:53:22.568129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.82:0/1568097096' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.192 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:23 smithi082 bash[79733]: audit 2023-10-09T04:53:22.568497+0000 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:23.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:23.347 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:23 smithi153 bash[59565]: audit 2023-10-09T04:53:22.568129+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.82:0/1568097096' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:23 smithi153 bash[59565]: audit 2023-10-09T04:53:22.568497+0000 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:53:23.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.488 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:23.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:23.626 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:23.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:23.751 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:23.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:23.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:23.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:23.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:24.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:24.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:24.146 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:24.249 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:24.279 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:24.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:22.822813+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1288: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:23.114011+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: audit 2023-10-09T04:53:23.117138+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:23.117260+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:23.119673+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:23.151317+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.395 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:24 smithi153 bash[59565]: cluster 2023-10-09T04:53:23.153556+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:24.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:22.822813+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1288: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:23.114011+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: audit 2023-10-09T04:53:23.117138+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:23.117260+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:23.119673+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:23.151317+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[79733]: cluster 2023-10-09T04:53:23.153556+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:22.822813+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1288: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:23.114011+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: audit 2023-10-09T04:53:23.117138+0000 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:53:24.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:23.117260+0000 mon.a (mon.0) 1733 : cluster 0 osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:53:24.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:23.119673+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:23.151317+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:24 smithi082 bash[77063]: cluster 2023-10-09T04:53:23.153556+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T04:53:24.520 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:24.553 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:24.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:24.677 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:24.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:24.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:24.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:24.922 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:24.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:24.986 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:53:24.986 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:53:25.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:25.045 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:25.148 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:25.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:25.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:25.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:25.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.115553+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.117588+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:53:25.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.120384+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.120632+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.370 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.120696+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.550747+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.115553+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.117588+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.120384+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.120632+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.120696+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.371 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:25 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.550747+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:25.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.115553+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.117588+0000 mon.a (mon.0) 1734 : cluster 0 osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:53:25.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.120384+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.120632+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.120696+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.418 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:25 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.550747+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T04:53:25.436 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:25.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:25.543 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:25.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:25.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:25.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:25.787 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:25.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:25.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T04:53:25.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:25.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:25.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:26.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:26.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:26.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:26.140 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:53:26.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[77063]: cluster 2023-10-09T04:53:24.823229+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1291: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:26.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[77063]: cluster 2023-10-09T04:53:25.132126+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[77063]: cluster 2023-10-09T04:53:25.133992+0000 mon.a (mon.0) 1735 : cluster 0 osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:53:26.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[77063]: audit 2023-10-09T04:53:25.578713+0000 mon.a (mon.0) 1736 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:53:26.168 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[77063]: cluster 2023-10-09T04:53:25.656501+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[79733]: cluster 2023-10-09T04:53:24.823229+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1291: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:26.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[79733]: cluster 2023-10-09T04:53:25.132126+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[79733]: cluster 2023-10-09T04:53:25.133992+0000 mon.a (mon.0) 1735 : cluster 0 osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:53:26.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[79733]: audit 2023-10-09T04:53:25.578713+0000 mon.a (mon.0) 1736 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:53:26.169 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:26 smithi082 bash[79733]: cluster 2023-10-09T04:53:25.656501+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:26.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:26.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:26.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:26 smithi153 bash[59565]: cluster 2023-10-09T04:53:24.823229+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1291: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:26 smithi153 bash[59565]: cluster 2023-10-09T04:53:25.132126+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:26 smithi153 bash[59565]: cluster 2023-10-09T04:53:25.133992+0000 mon.a (mon.0) 1735 : cluster 0 osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:53:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:26 smithi153 bash[59565]: audit 2023-10-09T04:53:25.578713+0000 mon.a (mon.0) 1736 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:53:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:26 smithi153 bash[59565]: cluster 2023-10-09T04:53:25.656501+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:26.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:26.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:26.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:26.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:26.598 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:26.703 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:26.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:26.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:26.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:26.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:26.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:27.012 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:27.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:53:27.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:27.144 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:27.158 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:53:27.182 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: cluster 2023-10-09T04:53:25.656376+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.182 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: cluster 2023-10-09T04:53:25.656443+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.182 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: audit 2023-10-09T04:53:26.135976+0000 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:53:27.182 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: cluster 2023-10-09T04:53:26.136082+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: cluster 2023-10-09T04:53:26.136516+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: audit 2023-10-09T04:53:26.350714+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.82:0/1430387382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[77063]: audit 2023-10-09T04:53:26.350993+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: cluster 2023-10-09T04:53:25.656376+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: cluster 2023-10-09T04:53:25.656443+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: audit 2023-10-09T04:53:26.135976+0000 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: cluster 2023-10-09T04:53:26.136082+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:53:27.183 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: cluster 2023-10-09T04:53:26.136516+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:27.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: audit 2023-10-09T04:53:26.350714+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.82:0/1430387382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.184 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:27 smithi082 bash[79733]: audit 2023-10-09T04:53:26.350993+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:27.275 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:27.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:27.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: cluster 2023-10-09T04:53:25.656376+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: cluster 2023-10-09T04:53:25.656443+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T04:53:27.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: audit 2023-10-09T04:53:26.135976+0000 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.82:0/2370989686' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:53:27.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: cluster 2023-10-09T04:53:26.136082+0000 mon.a (mon.0) 1738 : cluster 0 osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:53:27.390 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: cluster 2023-10-09T04:53:26.136516+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:27.390 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: audit 2023-10-09T04:53:26.350714+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.82:0/1430387382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.390 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:27 smithi153 bash[59565]: audit 2023-10-09T04:53:26.350993+0000 mon.a (mon.0) 1739 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:27.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:27.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:27.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:27.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:27.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:27.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:27.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:27.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:27.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:53:27.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:27.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:27.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:28.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:28.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:28.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:28.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:28.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:28.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:26.824087+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1294: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.108017+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.110984+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.113154+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.141731+0000 mon.a (mon.0) 1740 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.141776+0000 mon.a (mon.0) 1741 : cluster 1 Cluster is now healthy 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.152117+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.153183+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: audit 2023-10-09T04:53:27.153305+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.153401+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.154463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.155336+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.279511+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:28 smithi153 bash[59565]: cluster 2023-10-09T04:53:27.282650+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.450 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:28.553 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:28.688 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:26.824087+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1294: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:28.688 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.108017+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.110984+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.113154+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.141731+0000 mon.a (mon.0) 1740 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.141776+0000 mon.a (mon.0) 1741 : cluster 1 Cluster is now healthy 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.152117+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.153183+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: audit 2023-10-09T04:53:27.153305+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.153401+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.154463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.689 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.155336+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.279511+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[77063]: cluster 2023-10-09T04:53:27.282650+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:26.824087+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1294: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.108017+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.110984+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.113154+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.690 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.141731+0000 mon.a (mon.0) 1740 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.141776+0000 mon.a (mon.0) 1741 : cluster 1 Cluster is now healthy 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.152117+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.153183+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: audit 2023-10-09T04:53:27.153305+0000 mon.a (mon.0) 1742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.153401+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.154463+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.155336+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.279511+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.691 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:28 smithi082 bash[79733]: cluster 2023-10-09T04:53:27.282650+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T04:53:28.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:28.726 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:28.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:28.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:28.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:28.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:28.966 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:28.998 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:29.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:29.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:29.114 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:29.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:29.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:29.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:29.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:29.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:29.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:29.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:29.642 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.669 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:29.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:29.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.922 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:29] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T04:53:29.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:29.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:30.044 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:30.180 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:30.212 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[77063]: cluster 2023-10-09T04:53:28.825013+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[77063]: audit 2023-10-09T04:53:29.283698+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[77063]: audit 2023-10-09T04:53:29.284189+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[77063]: audit 2023-10-09T04:53:29.300485+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[77063]: audit 2023-10-09T04:53:29.302062+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[79733]: cluster 2023-10-09T04:53:28.825013+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[79733]: audit 2023-10-09T04:53:29.283698+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[79733]: audit 2023-10-09T04:53:29.284189+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2023-10-09T04:53:30.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[79733]: audit 2023-10-09T04:53:29.300485+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:30.233 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:30 smithi082 bash[79733]: audit 2023-10-09T04:53:29.302062+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:30.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:30 smithi153 bash[59565]: cluster 2023-10-09T04:53:28.825013+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:30.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:30 smithi153 bash[59565]: audit 2023-10-09T04:53:29.283698+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-09T04:53:30.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:30 smithi153 bash[59565]: audit 2023-10-09T04:53:29.284189+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]: dispatch 2023-10-09T04:53:30.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:30 smithi153 bash[59565]: audit 2023-10-09T04:53:29.300485+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:30.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:30 smithi153 bash[59565]: audit 2023-10-09T04:53:29.302062+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:30.346 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:30.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:30.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:30.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:30.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:30.611 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:30.693 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:53:30.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:30.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:30.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:30.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:30.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:30.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:31.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:31.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:31.128 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:31.158 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:31.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:31.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:31.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:31.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:31 smithi153 bash[59565]: cluster 2023-10-09T04:53:30.152839+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:31 smithi153 bash[59565]: cluster 2023-10-09T04:53:30.153056+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:31 smithi153 bash[59565]: audit 2023-10-09T04:53:30.155386+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T04:53:31.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:31 smithi153 bash[59565]: audit 2023-10-09T04:53:30.155485+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2023-10-09T04:53:31.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:31 smithi153 bash[59565]: cluster 2023-10-09T04:53:30.155566+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:53:31.429 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:31.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[77063]: cluster 2023-10-09T04:53:30.152839+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[77063]: cluster 2023-10-09T04:53:30.153056+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[77063]: audit 2023-10-09T04:53:30.155386+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[77063]: audit 2023-10-09T04:53:30.155485+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[77063]: cluster 2023-10-09T04:53:30.155566+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[79733]: cluster 2023-10-09T04:53:30.152839+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[79733]: cluster 2023-10-09T04:53:30.153056+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[79733]: audit 2023-10-09T04:53:30.155386+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[79733]: audit 2023-10-09T04:53:30.155485+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2023-10-09T04:53:31.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:31 smithi082 bash[79733]: cluster 2023-10-09T04:53:30.155566+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:53:31.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:31.563 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:31.666 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:31.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:31.801 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:31.831 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:31.859 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:31.935 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:31.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:31.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:32.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:32.090 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:32.189 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:32.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:32.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:32.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:32.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:32 smithi153 bash[59565]: cluster 2023-10-09T04:53:30.533897+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:32.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:32 smithi153 bash[59565]: cluster 2023-10-09T04:53:30.825464+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:32.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:32 smithi153 bash[59565]: cluster 2023-10-09T04:53:31.166871+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:32 smithi153 bash[59565]: cluster 2023-10-09T04:53:31.167181+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.332 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:32 smithi153 bash[59565]: cluster 2023-10-09T04:53:31.170798+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:53:32.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:32.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:32.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:32.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[79733]: cluster 2023-10-09T04:53:30.533897+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:32.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[79733]: cluster 2023-10-09T04:53:30.825464+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:32.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[79733]: cluster 2023-10-09T04:53:31.166871+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[79733]: cluster 2023-10-09T04:53:31.167181+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[79733]: cluster 2023-10-09T04:53:31.170798+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[77063]: cluster 2023-10-09T04:53:30.533897+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[77063]: cluster 2023-10-09T04:53:30.825464+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[77063]: cluster 2023-10-09T04:53:31.166871+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[77063]: cluster 2023-10-09T04:53:31.167181+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T04:53:32.514 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:32 smithi082 bash[77063]: cluster 2023-10-09T04:53:31.170798+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:53:32.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:32.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:32.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:32.741 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:32.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:32.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:32.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:32.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:33.009 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:33.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:33.140 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:33.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:33.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:33.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:33.375 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:33.401 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:33.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:33.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:33.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:33.637 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:33.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:33.774 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:33.800 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:33.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:33.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:33.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:34.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:34.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:34.195 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:34.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:34.271 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:53:34.296 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:53:34.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:34.307 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:34 smithi082 bash[79733]: cluster 2023-10-09T04:53:32.826187+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:34.307 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:34 smithi082 bash[77063]: cluster 2023-10-09T04:53:32.826187+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:34.320 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:53:34.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:34.349 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:53:34.374 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:53:34.396 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:53:34.418 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:53:34.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:34 smithi153 bash[59565]: cluster 2023-10-09T04:53:32.826187+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:34.441 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.443 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:53:34.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:34.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:34.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:34.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:34.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:34.931 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:34.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:34.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:35.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:35.085 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:35.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:35.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:35.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:35 smithi082 bash[77063]: audit 2023-10-09T04:53:34.243015+0000 mon.a (mon.0) 1752 : audit 0 from='client.? 172.21.15.82:0/3394358610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:35.202 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:35 smithi082 bash[77063]: audit 2023-10-09T04:53:35.013894+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:35.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:35 smithi082 bash[79733]: audit 2023-10-09T04:53:34.243015+0000 mon.a (mon.0) 1752 : audit 0 from='client.? 172.21.15.82:0/3394358610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:35.202 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:35 smithi082 bash[79733]: audit 2023-10-09T04:53:35.013894+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:35.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:35.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:35.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:35 smithi153 bash[59565]: audit 2023-10-09T04:53:34.243015+0000 mon.a (mon.0) 1752 : audit 0 from='client.? 172.21.15.82:0/3394358610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:35.331 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:35 smithi153 bash[59565]: audit 2023-10-09T04:53:35.013894+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:53:35.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:35.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:35.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:35.585 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:35.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:35.711 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:35.733 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:35.783 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:53:35.784 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:53:35.784 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T04:53:35.785 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:53:35.837 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:35.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:35.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:35.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:35.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:36.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:36.133 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:36.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:36.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:36.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:36.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:36 smithi153 bash[59565]: cluster 2023-10-09T04:53:34.826918+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:36.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:36 smithi082 bash[77063]: cluster 2023-10-09T04:53:34.826918+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:36.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:36 smithi082 bash[79733]: cluster 2023-10-09T04:53:34.826918+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1301: 89 pgs: 89 active+clean; 4.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:36.496 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:36.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:36.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:36.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:36.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:36.788 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:36.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:53:36.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:36.916 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:36.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:37.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:37.041 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:37.145 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:37.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:37.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:37.298 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:37.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:37.401 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:37.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.578 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:37.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:37.707 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:37.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:37.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:37.974 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:37.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:37.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:38.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:38.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:38.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:38.229 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:38 smithi153 bash[59565]: cluster 2023-10-09T04:53:36.827977+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:38.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:38.298 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:38.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:38.387 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:38.448 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:38 smithi082 bash[77063]: cluster 2023-10-09T04:53:36.827977+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:38.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:38 smithi082 bash[79733]: cluster 2023-10-09T04:53:36.827977+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:53:38.491 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:38.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:38.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:38.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:38.642 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:38.745 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:38.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:38.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:38.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:38.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:38.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:39.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:39.039 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:39.143 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:39.174 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:39.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:39.309 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:39.322 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:39 smithi082 bash[77063]: cluster 2023-10-09T04:53:38.828876+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:39.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:39.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:39 smithi153 bash[59565]: cluster 2023-10-09T04:53:38.828876+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:39.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:39.550 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:39.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:39.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:39.709 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:39.711 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:39 smithi082 bash[79733]: cluster 2023-10-09T04:53:38.828876+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:39.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:53:39.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:39.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:39.949 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:39.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:39.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:39.996 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:39] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T04:53:40.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:40.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.216 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:40.318 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:40.351 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:40.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.479 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:40.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:40.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:40.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:40.715 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:40.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:40.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:40.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:40.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:41.012 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:41.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:41.116 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:41.141 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:41.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:41.271 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:41.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:41.374 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:41.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:41.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:41.502 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:41.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:41.602 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:41.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:41.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:41.761 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:41.787 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:41.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:41.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:41.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:42.025 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.052 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:42.118 INFO:teuthology.orchestra.run.smithi082.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T04:53:42.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:42 smithi153 bash[59565]: cluster 2023-10-09T04:53:40.829734+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:42.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:42 smithi153 bash[59565]: audit 2023-10-09T04:53:41.833217+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:42.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.295 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:42 smithi082 bash[79733]: cluster 2023-10-09T04:53:40.829734+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:42.295 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:42 smithi082 bash[79733]: audit 2023-10-09T04:53:41.833217+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.295 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:42 smithi082 bash[77063]: cluster 2023-10-09T04:53:40.829734+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:42.295 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:42 smithi082 bash[77063]: audit 2023-10-09T04:53:41.833217+0000 mon.a (mon.0) 1754 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:42.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:42.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:42.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:42.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:42.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:42.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:42.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:42.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:42.818 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.845 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:42.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:42.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:42.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:43.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:43.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:43 smithi153 bash[59565]: audit 2023-10-09T04:53:42.111894+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:43 smithi153 bash[59565]: cluster 2023-10-09T04:53:42.111976+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:53:43.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:43 smithi153 bash[59565]: cluster 2023-10-09T04:53:42.112034+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.215 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:43 smithi153 bash[59565]: cluster 2023-10-09T04:53:42.112343+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:43.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:43.363 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:43.466 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.490 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[77063]: audit 2023-10-09T04:53:42.111894+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[77063]: cluster 2023-10-09T04:53:42.111976+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[77063]: cluster 2023-10-09T04:53:42.112034+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[77063]: cluster 2023-10-09T04:53:42.112343+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[79733]: audit 2023-10-09T04:53:42.111894+0000 mon.a (mon.0) 1755 : audit 1 from='client.? 172.21.15.82:0/783702661' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.506 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[79733]: cluster 2023-10-09T04:53:42.111976+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:53:43.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[79733]: cluster 2023-10-09T04:53:42.112034+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.507 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:43 smithi082 bash[79733]: cluster 2023-10-09T04:53:42.112343+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:43.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:43.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:43.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:43.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:43.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:43.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:43.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:44.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:44.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:44.113 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:44.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:44.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.250 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:44 smithi153 bash[59565]: cluster 2023-10-09T04:53:42.830242+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:44.250 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:44 smithi153 bash[59565]: cluster 2023-10-09T04:53:43.055015+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:44.267 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:44.370 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:44.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:44 smithi082 bash[79733]: cluster 2023-10-09T04:53:42.830242+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:44.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:44 smithi082 bash[79733]: cluster 2023-10-09T04:53:43.055015+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:44.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:44 smithi082 bash[77063]: cluster 2023-10-09T04:53:42.830242+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:44.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:44 smithi082 bash[77063]: cluster 2023-10-09T04:53:43.055015+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T04:53:44.502 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:44.528 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:44.632 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:44.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:44.764 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:44.795 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:44.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:44.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:44.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:45.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:45.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:45.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:45.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:45.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:45 smithi082 bash[79733]: audit 2023-10-09T04:53:44.297150+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:45.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:45 smithi082 bash[77063]: audit 2023-10-09T04:53:44.297150+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:45.286 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:45 smithi153 bash[59565]: audit 2023-10-09T04:53:44.297150+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:45.292 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:45.319 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:45.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:45.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:45.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:45.557 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:53:45.558 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:53:45.558 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:53:45.558 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:53:45.558 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:53:45.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:45.674 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:45.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:45.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.956 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:45.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:45.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:46.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:46.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:46.144 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:53:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:46.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:46 smithi082 bash[77063]: cluster 2023-10-09T04:53:44.830865+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:46.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:46 smithi082 bash[77063]: audit 2023-10-09T04:53:45.931266+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:53:46.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:46 smithi082 bash[79733]: cluster 2023-10-09T04:53:44.830865+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:46.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:46 smithi082 bash[79733]: audit 2023-10-09T04:53:45.931266+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:53:46.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:46.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:46.350 DEBUG:teuthology.orchestra.run.smithi082:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T04:53:46.353 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:53:46.353 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:53:46.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:46.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:hints = 1 2023-10-09T04:53:46.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:53:46.389 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object prefix: benchmark_data_smithi082_223129 2023-10-09T04:53:46.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:53:46.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:53:46.400 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:46 smithi153 bash[59565]: cluster 2023-10-09T04:53:44.830865+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:53:46.401 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:46 smithi153 bash[59565]: audit 2023-10-09T04:53:45.931266+0000 mon.a (mon.0) 1758 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:53:46.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:46.481 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:53:46.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:46.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:46.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:46.688 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:46.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:46.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:46.926 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:46.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:46.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:47.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:47.081 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:47.184 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.136251+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.136502+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: audit 2023-10-09T04:53:46.138553+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:53:47.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.138602+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.392163+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.392710+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.136251+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.136502+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: audit 2023-10-09T04:53:46.138553+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.138602+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.392163+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.196 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:47 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.392710+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.215 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:47.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:47.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:47.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:47.390 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 1 15 1998 1983 123.934 123.938 0.00456168 0.0080487 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.136251+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.136502+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: audit 2023-10-09T04:53:46.138553+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.82:0/960255965' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.138602+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.392163+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:47 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.392710+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T04:53:47.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:47.485 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:47.590 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:47.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:47.744 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:47.848 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:47.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:47.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:47.978 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:48.005 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:48.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:48.135 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:48.152 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:48 smithi082 bash[77063]: cluster 2023-10-09T04:53:46.831888+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 12 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:53:48.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:48.248 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:48 smithi153 bash[59565]: cluster 2023-10-09T04:53:46.831888+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 12 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:53:48.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:48.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:48.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:48.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 2 16 4764 4748 148.287 172.812 0.0109221 0.00671695 2023-10-09T04:53:48.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:48.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:48 smithi082 bash[79733]: cluster 2023-10-09T04:53:46.831888+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 12 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:53:48.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:48.528 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:48.633 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:48.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:48.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:48.791 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:48.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:48.925 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:48.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:49.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:49.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:49.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:49.186 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:49.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.326 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:49.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 3 16 7373 7357 153.204 163.062 0.0018676 0.00644746 2023-10-09T04:53:49.430 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:49.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:49.563 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:49.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:49.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:49] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.33.4" 2023-10-09T04:53:49.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:49.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:49.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:49.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:49.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:50.090 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:50.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:50.235 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:50 smithi153 bash[59565]: cluster 2023-10-09T04:53:48.832823+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 184 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:53:50.250 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:50.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:50.354 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:50.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:50.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 4 16 9938 9922 154.975 160.312 0.00540725 0.00644232 2023-10-09T04:53:50.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:50.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:50.604 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:50 smithi082 bash[77063]: cluster 2023-10-09T04:53:48.832823+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 184 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:53:50.604 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:50 smithi082 bash[79733]: cluster 2023-10-09T04:53:48.832823+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 184 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:53:50.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:50.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:50.781 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:50.887 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.917 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:50.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:50.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:51.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.053 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:51.157 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:51.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:51.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:51.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:53:51.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.318 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:51.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:51.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 5 16 11779 11763 146.992 115.062 0.00402123 0.00678927 2023-10-09T04:53:51.437 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:51.471 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:51.575 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:51.612 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:51.716 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.742 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:51.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:51.873 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:51.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:51.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:52.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:52.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:52.133 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:52.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:52 smithi082 bash[77063]: cluster 2023-10-09T04:53:50.833367+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 252 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 454 op/s 2023-10-09T04:53:52.159 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:52 smithi082 bash[77063]: audit 2023-10-09T04:53:51.988762+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:52.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:52 smithi082 bash[79733]: cluster 2023-10-09T04:53:50.833367+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 252 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 454 op/s 2023-10-09T04:53:52.160 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:52 smithi082 bash[79733]: audit 2023-10-09T04:53:51.988762+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:52.187 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:53:52.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:52.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:52.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:52.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:52.377 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:52 smithi153 bash[59565]: cluster 2023-10-09T04:53:50.833367+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 252 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 454 op/s 2023-10-09T04:53:52.377 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:52 smithi153 bash[59565]: audit 2023-10-09T04:53:51.988762+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:52.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 6 15 13686 13671 142.367 119.25 0.00716426 0.00701664 2023-10-09T04:53:52.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:52.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:52.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:52.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:52.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:52.655 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:52.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:52.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:52.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:52.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:52.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:53.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.046 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:53.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:53.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.155830+0000 mon.a (mon.0) 1762 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:53.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.169619+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.161 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.172722+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.162 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[77063]: audit 2023-10-09T04:53:52.179406+0000 mon.a (mon.0) 1763 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:53.162 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.179478+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:53:53.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:53.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:53.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:53.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 7 16 15881 15865 141.617 137.125 0.0147203 0.00705488 2023-10-09T04:53:53.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:53 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.155830+0000 mon.a (mon.0) 1762 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:53.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:53 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.169619+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:53 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.172722+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:53 smithi153 bash[59565]: audit 2023-10-09T04:53:52.179406+0000 mon.a (mon.0) 1763 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:53.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:53 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.179478+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:53:53.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:53.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.155830+0000 mon.a (mon.0) 1762 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.169619+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.172722+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[79733]: audit 2023-10-09T04:53:52.179406+0000 mon.a (mon.0) 1763 : audit 1 from='client.? 172.21.15.82:0/1115788983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:53.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:53 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.179478+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:53:53.540 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:53.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:53.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:53.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:53.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:53.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:53.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.978 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:53.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:54.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:54.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.241 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:54.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:54.353 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:54 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.175013+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:54.353 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:54 smithi153 bash[59565]: cluster 2023-10-09T04:53:52.833895+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 571 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:54.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:54.391 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 8 16 17839 17823 139.21 122.375 0.00386733 0.0071767 2023-10-09T04:53:54.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:54 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.175013+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:54 smithi082 bash[77063]: cluster 2023-10-09T04:53:52.833895+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 571 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:54 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.175013+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T04:53:54.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:54 smithi082 bash[79733]: cluster 2023-10-09T04:53:52.833895+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1313: 89 pgs: 89 active+clean; 571 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:53:54.503 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:54.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:54.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:54.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:54.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:54.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:54.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:54.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:55.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:55.133 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:55.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:55.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:55.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 9 16 19864 19848 137.804 126.562 0.00809846 0.00725039 2023-10-09T04:53:55.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:55.416 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:55.520 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.547 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:55.650 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:55.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:55.780 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:55.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:55.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:55.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:55.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:56.047 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:56.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:56 smithi082 bash[77063]: cluster 2023-10-09T04:53:54.834340+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 861 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:53:56.167 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:56 smithi082 bash[79733]: cluster 2023-10-09T04:53:54.834340+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 861 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:53:56.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:56.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:56.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:56 smithi153 bash[59565]: cluster 2023-10-09T04:53:54.834340+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1314: 89 pgs: 89 active+clean; 861 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:53:56.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:56.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 10 16 21601 21585 134.879 108.562 0.00662983 0.00740745 2023-10-09T04:53:56.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:56.470 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:56.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:56.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:56.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:56.698 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:56.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:56.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:56.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:56.973 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:56.997 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:57.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:57.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:57.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:57.258 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:57.362 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:57.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:57.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 11 16 22793 22777 129.39 74.5 0.00620603 0.00772134 2023-10-09T04:53:57.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:57.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:57.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:57.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:57.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:57.660 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:53:57.706 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:53:57.743 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:53:57.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:57.784 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:57.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:57.789 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:53:57.827 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:53:57.856 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:53:57.888 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:57.894 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:53:57.912 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:57.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:57.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:57.929 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:53:57.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.932 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:53:57.933 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:53:58.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:58.143 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:58.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:58 smithi082 bash[77063]: cluster 2023-10-09T04:53:56.834999+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 1017 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:53:58.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:53:58 smithi082 bash[77063]: audit 2023-10-09T04:53:57.617173+0000 mon.a (mon.0) 1765 : audit 0 from='client.? 172.21.15.82:0/2628189625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:58 smithi082 bash[79733]: cluster 2023-10-09T04:53:56.834999+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 1017 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:53:58.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:53:58 smithi082 bash[79733]: audit 2023-10-09T04:53:57.617173+0000 mon.a (mon.0) 1765 : audit 0 from='client.? 172.21.15.82:0/2628189625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:58.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:58.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:53:58.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 12 16 24531 24515 127.659 108.625 0.00435973 0.0078272 2023-10-09T04:53:58.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:58 smithi153 bash[59565]: cluster 2023-10-09T04:53:56.834999+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 1017 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:53:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:53:58 smithi153 bash[59565]: audit 2023-10-09T04:53:57.617173+0000 mon.a (mon.0) 1765 : audit 0 from='client.? 172.21.15.82:0/2628189625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:58.427 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:58.532 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:58.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:58.597 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:53:58.597 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:53:58.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:58.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:58.806 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:58.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:58.939 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:58.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:58.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:59.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:59.090 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:59.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:59.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:59.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:59.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 13 16 26282 26266 126.257 109.438 0.00650024 0.00791382 2023-10-09T04:53:59.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.486 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:59.589 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:59.618 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:59.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:59.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:59.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:59.867 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:53:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:53:59] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.33.4" 2023-10-09T04:53:59.882 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:53:59.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:59.986 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:00.012 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:00.115 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:00.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:00.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[77063]: cluster 2023-10-09T04:53:58.835845+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:00.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[77063]: audit 2023-10-09T04:53:59.303224+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:00.178 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[77063]: audit 2023-10-09T04:53:59.304538+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:00.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:54:00.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:00.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:00 smithi153 bash[59565]: cluster 2023-10-09T04:53:58.835845+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:00.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:00 smithi153 bash[59565]: audit 2023-10-09T04:53:59.303224+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:00.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:00 smithi153 bash[59565]: audit 2023-10-09T04:53:59.304538+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:00.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:00.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:00.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 14 16 28090 28074 125.309 113 0.00460722 0.00797333 2023-10-09T04:54:00.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[79733]: cluster 2023-10-09T04:53:58.835845+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[79733]: audit 2023-10-09T04:53:59.303224+0000 mon.a (mon.0) 1766 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:00 smithi082 bash[79733]: audit 2023-10-09T04:53:59.304538+0000 mon.a (mon.0) 1767 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:00.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:00.553 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:00.657 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:00.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:00.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:00.824 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:00.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:00.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:00.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:01.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:01.093 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:01.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:01.203 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:01.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:01.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:01.382 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:01.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 15 16 29769 29753 123.95 104.938 0.0109032 0.00806162 2023-10-09T04:54:01.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:01.516 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:01.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:54:01.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:01.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:01.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:01.785 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:01.816 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:01.919 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:01.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:01.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:02.049 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:02.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:02.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:02.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:02.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:02.319 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:02 smithi153 bash[59565]: cluster 2023-10-09T04:54:00.837330+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 1.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:02.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:02.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 16 16 31376 31360 122.48 100.438 0.0118277 0.00815886 2023-10-09T04:54:02.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:02.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:02 smithi082 bash[77063]: cluster 2023-10-09T04:54:00.837330+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 1.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:02.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:02 smithi082 bash[79733]: cluster 2023-10-09T04:54:00.837330+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 1.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:54:02.472 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:02.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:02.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:02.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:02.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:02.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:02.837 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:02.862 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:02.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:02.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:02.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:02.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:03.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:03.102 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:03.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:03.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:03.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:03.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:03.366 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:03.374 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:03 smithi153 bash[59565]: cluster 2023-10-09T04:54:02.838343+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:54:03.388 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:03.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 17 16 33028 33012 121.349 103.25 0.0140513 0.00823481 2023-10-09T04:54:03.494 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:03.503 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:03 smithi082 bash[77063]: cluster 2023-10-09T04:54:02.838343+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:54:03.503 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:03 smithi082 bash[79733]: cluster 2023-10-09T04:54:02.838343+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 1.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:54:03.517 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:03.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:03.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:03.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:03.783 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:03.885 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:03.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:03.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:03.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:04.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:04.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:04.139 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:04.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:04.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:04.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:04.310 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:04.392 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 18 16 34544 34528 119.871 94.75 0.00486043 0.00833433 2023-10-09T04:54:04.414 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:04.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:04.551 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:04.582 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:04.686 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:04.717 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:04.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:04.850 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:04.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:04.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:04.953 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:04.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:05.044 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:54:05.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:05.142 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:05.244 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:05.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:05.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:05.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:05.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 19 16 35356 35340 116.233 50.75 0.0143961 0.00859674 2023-10-09T04:54:05.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:05.507 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:05.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:05.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:05.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:05.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:05.801 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:05.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:05.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:05.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:06.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:06.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:06.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:06.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:06.221 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:06 smithi082 bash[77063]: cluster 2023-10-09T04:54:04.838861+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:54:06.222 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:06 smithi082 bash[79733]: cluster 2023-10-09T04:54:04.838861+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:54:06.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:06.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:06.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:54:06.392516+0000 min lat: 0.00123875 max lat: 0.508003 avg lat: 0.0086676 2023-10-09T04:54:06.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:06.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 20 15 36914 36899 115.292 97.4375 0.00466046 0.0086676 2023-10-09T04:54:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:06 smithi153 bash[59565]: cluster 2023-10-09T04:54:04.838861+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:54:06.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:06.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:06.569 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:06.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:06.690 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:06.700 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:06.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:06.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:06.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:06.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:06.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:06.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:06.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:07.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:07.120 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:07.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:07.225 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:07.262 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:07.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:07.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:07.383 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:07.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 21 16 38268 38252 113.829 84.5625 0.00836069 0.00876304 2023-10-09T04:54:07.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:07.518 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:07.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:07.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:07.758 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:07.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:07.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:07.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:07.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:07.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:08.021 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:08.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:08.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:08.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:08.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:08.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:08.381 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:08 smithi153 bash[59565]: cluster 2023-10-09T04:54:06.839405+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:54:08.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 22 16 39781 39765 112.953 94.5625 0.00615381 0.00884788 2023-10-09T04:54:08.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:08.423 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:08 smithi082 bash[77063]: cluster 2023-10-09T04:54:06.839405+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:54:08.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:08 smithi082 bash[79733]: cluster 2023-10-09T04:54:06.839405+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:54:08.440 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:08.544 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:08.675 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.703 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:08.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:08.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:08.934 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:08.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:08.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:09.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:09.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:09.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:09.326 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.352 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:09.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 23 16 41161 41145 111.791 86.25 0.0653986 0.00892432 2023-10-09T04:54:09.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:09.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:09.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:09.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:09.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:09.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:09.858 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:09.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:09.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:09.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:09] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.33.4" 2023-10-09T04:54:09.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.023 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:10.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:10.134 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:10 smithi153 bash[59565]: cluster 2023-10-09T04:54:08.840115+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:54:10.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:10.259 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:10.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:10.386 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 24 16 42659 42643 111.034 93.625 0.00941436 0.00900068 2023-10-09T04:54:10.418 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.419 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:10 smithi082 bash[77063]: cluster 2023-10-09T04:54:08.840115+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:54:10.420 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:10 smithi082 bash[79733]: cluster 2023-10-09T04:54:08.840115+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:54:10.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:10.522 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:10.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:10.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:10.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:10.820 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:10.945 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:10.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:10.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:11.086 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:11.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:11.157 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:54:11.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:11.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:11 smithi082 bash[77063]: audit 2023-10-09T04:54:11.127353+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.82:0/3673453926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.228 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:11 smithi082 bash[77063]: audit 2023-10-09T04:54:11.128794+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:11 smithi082 bash[79733]: audit 2023-10-09T04:54:11.127353+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.82:0/3673453926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:11 smithi082 bash[79733]: audit 2023-10-09T04:54:11.128794+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.245 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:11.348 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:11.359 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:11 smithi153 bash[59565]: audit 2023-10-09T04:54:11.127353+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.82:0/3673453926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.359 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:11 smithi153 bash[59565]: audit 2023-10-09T04:54:11.128794+0000 mon.a (mon.0) 1768 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:54:11.376 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:11.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 25 16 43863 43847 109.602 75.25 0.0127188 0.0091179 2023-10-09T04:54:11.482 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:11.506 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:11.609 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:11.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:11.671 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:54:11.674 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:54:11.750 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:11.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:11.898 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:11.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:11.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:12.028 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:12.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:12.153 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: cluster 2023-10-09T04:54:10.842451+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: cluster 2023-10-09T04:54:11.139155+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: cluster 2023-10-09T04:54:11.142037+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.143249+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: cluster 2023-10-09T04:54:11.143404+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.407032+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.887014+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.895558+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.902997+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.154 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[77063]: audit 2023-10-09T04:54:11.910712+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.164 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:12.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:12.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: cluster 2023-10-09T04:54:10.842451+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: cluster 2023-10-09T04:54:11.139155+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: cluster 2023-10-09T04:54:11.142037+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.143249+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: cluster 2023-10-09T04:54:11.143404+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.407032+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:12.316 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.887014+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.895558+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.902997+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.317 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:12 smithi153 bash[59565]: audit 2023-10-09T04:54:11.910712+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:12.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 26 16 45035 45019 108.204 73.25 0.00513641 0.00923448 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: cluster 2023-10-09T04:54:10.842451+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: cluster 2023-10-09T04:54:11.139155+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: cluster 2023-10-09T04:54:11.142037+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.143249+0000 mon.a (mon.0) 1769 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: cluster 2023-10-09T04:54:11.143404+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.407032+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.887014+0000 mon.a (mon.0) 1772 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.895558+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.902997+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.409 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:12 smithi082 bash[79733]: audit 2023-10-09T04:54:11.910712+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:12.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:12.466 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:12.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:12.572 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:12.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:12.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:12.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:12.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:12.969 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:13.004 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:13.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:13.136 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:13.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:54:13.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:13.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:13.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:13.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 27 16 46277 46261 107.071 77.625 0.00922718 0.00933312 2023-10-09T04:54:13.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:13.505 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:13.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:13.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:13.673 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:13.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:13.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:13.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:13.944 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:13.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:14.048 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:14.076 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:14.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:14.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:14.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:14 smithi082 bash[77063]: cluster 2023-10-09T04:54:12.843398+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T04:54:14.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:14 smithi082 bash[79733]: cluster 2023-10-09T04:54:12.843398+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T04:54:14.310 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:14.320 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:14 smithi153 bash[59565]: cluster 2023-10-09T04:54:12.843398+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1324: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T04:54:14.339 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:14.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 28 16 47668 47652 106.352 86.9375 0.0104822 0.0093974 2023-10-09T04:54:14.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:14.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:14.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:14.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:14.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:14.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:14.828 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:14.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:14.955 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:14.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:15.062 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:15.091 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:15.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:15.227 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:15.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:15.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:15.352 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:15.393 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 29 16 48955 48939 105.458 80.4375 0.012925 0.00947697 2023-10-09T04:54:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:15 smithi153 bash[59565]: audit 2023-10-09T04:54:14.297758+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:15.455 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:15 smithi082 bash[77063]: audit 2023-10-09T04:54:14.297758+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:15.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:15 smithi082 bash[79733]: audit 2023-10-09T04:54:14.297758+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:15.483 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:15.587 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:15.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:15.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:15.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:15.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:15.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:15.883 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:15.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:15.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:16.013 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:16.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:16.118 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:54:16.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:16.147 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:16.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:16.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:16.281 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:16.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:16.389 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:16.631 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 30 16 50185 50169 104.504 76.875 0.0124271 0.00956272 2023-10-09T04:54:16.631 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:16 smithi082 bash[77063]: cluster 2023-10-09T04:54:14.843984+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:54:16.632 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:16 smithi082 bash[79733]: cluster 2023-10-09T04:54:14.843984+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:54:16.632 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:16 smithi153 bash[59565]: cluster 2023-10-09T04:54:14.843984+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-10-09T04:54:16.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:16.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:16.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:17.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:17.415 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 31 16 51342 51326 103.465 72.3125 0.00353151 0.00962823 2023-10-09T04:54:17.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:17.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:17.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:17.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:17.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:17.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:18.279 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:54:18.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:18.394 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 32 16 52445 52429 102.386 68.9375 0.0094387 0.00976157 2023-10-09T04:54:18.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:18.441 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:18 smithi082 bash[79733]: cluster 2023-10-09T04:54:16.844590+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:18.442 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:18 smithi082 bash[77063]: cluster 2023-10-09T04:54:16.844590+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:18.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:18.539 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:54:18.562 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:18.571 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:18 smithi153 bash[59565]: cluster 2023-10-09T04:54:16.844590+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:54:18.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:18.605 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:54:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:18.649 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:54:18.698 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:54:18.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:18.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:18.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:54:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:18.767 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:54:18.805 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:54:18.835 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:18.840 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:54:18.844 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.844 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.845 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:54:18.864 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:18.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:18.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:18.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:19.100 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:19.135 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:19.240 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:19.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:19.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:19.323 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[77063]: audit 2023-10-09T04:54:18.325121+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.323 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[77063]: audit 2023-10-09T04:54:18.335005+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.323 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[77063]: audit 2023-10-09T04:54:18.374220+0000 mon.a (mon.0) 1779 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[77063]: audit 2023-10-09T04:54:18.476967+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.82:0/596759826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:19.324 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[77063]: cluster 2023-10-09T04:54:18.845278+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:54:19.352 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:54:19.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:19.395 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 33 16 53625 53609 101.518 73.75 0.0131019 0.00984279 2023-10-09T04:54:19.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:19.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:19.537 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:19.643 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:19.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:19 smithi153 bash[59565]: audit 2023-10-09T04:54:18.325121+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:19 smithi153 bash[59565]: audit 2023-10-09T04:54:18.335005+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:19 smithi153 bash[59565]: audit 2023-10-09T04:54:18.374220+0000 mon.a (mon.0) 1779 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:19 smithi153 bash[59565]: audit 2023-10-09T04:54:18.476967+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.82:0/596759826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:19.652 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:19 smithi153 bash[59565]: cluster 2023-10-09T04:54:18.845278+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:54:19.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:19.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[79733]: audit 2023-10-09T04:54:18.325121+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[79733]: audit 2023-10-09T04:54:18.335005+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:19.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[79733]: audit 2023-10-09T04:54:18.374220+0000 mon.a (mon.0) 1779 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[79733]: audit 2023-10-09T04:54:18.476967+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.82:0/596759826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[79733]: cluster 2023-10-09T04:54:18.845278+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1327: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:54:19.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:19] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-10-09T04:54:19.772 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:19.808 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:19.912 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:19.940 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:19.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:19.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:20.044 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:20.070 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:20.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:20.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:20.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:20.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:20.394 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 34 16 54809 54793 100.709 74 0.0127977 0.00991977 2023-10-09T04:54:20.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:20.456 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:20.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: cluster 2023-10-09T04:54:19.335353+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: cluster 2023-10-09T04:54:19.340131+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: cluster 2023-10-09T04:54:19.341813+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: cluster 2023-10-09T04:54:19.346070+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: audit 2023-10-09T04:54:19.346207+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[77063]: cluster 2023-10-09T04:54:19.346284+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: cluster 2023-10-09T04:54:19.335353+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: cluster 2023-10-09T04:54:19.340131+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: cluster 2023-10-09T04:54:19.341813+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: cluster 2023-10-09T04:54:19.346070+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: audit 2023-10-09T04:54:19.346207+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:20.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:20 smithi082 bash[79733]: cluster 2023-10-09T04:54:19.346284+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:54:20.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:20.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:20.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: cluster 2023-10-09T04:54:19.335353+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:20.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: cluster 2023-10-09T04:54:19.340131+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: cluster 2023-10-09T04:54:19.341813+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: cluster 2023-10-09T04:54:19.346070+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T04:54:20.662 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: audit 2023-10-09T04:54:19.346207+0000 mon.a (mon.0) 1781 : audit 1 from='client.? 172.21.15.82:0/331187221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:20.663 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:20 smithi153 bash[59565]: cluster 2023-10-09T04:54:19.346284+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:54:20.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:20.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:20.833 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:20.865 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:20.969 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:20.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:20.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:21.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.124 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:21.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:21.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:21.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:21.394 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 35 16 56003 55987 99.9634 74.625 0.0185644 0.00999715 2023-10-09T04:54:21.397 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:21.501 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.528 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:21.570 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:21 smithi082 bash[77063]: cluster 2023-10-09T04:54:20.845863+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:54:21.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:21 smithi153 bash[59565]: cluster 2023-10-09T04:54:20.845863+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:54:21.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:21.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:21.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:21.901 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:21 smithi082 bash[79733]: cluster 2023-10-09T04:54:20.845863+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1329: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:54:21.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:21.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:21.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:22.030 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:22.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:22.155 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:22.180 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:22.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:22.307 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:22.395 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 36 16 57178 57162 99.2245 73.4375 0.0157679 0.0100719 2023-10-09T04:54:22.411 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:22.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:22.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:22.579 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:22.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:22.720 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:22.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:22.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:22.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:22.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:22.963 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:22.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:22.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:23.092 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:23.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:23.201 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:54:23.287 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:23.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:23.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:23.398 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 37 16 58136 58120 98.1599 59.875 0.0173852 0.0101818 2023-10-09T04:54:23.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:23.477 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:23.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:23.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:23.609 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:23.653 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:54:23.653 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:54:23.653 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T04:54:23.653 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T04:54:23.653 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:54:23.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:23.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:23.844 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:23.875 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:23.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:23.936 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:54:23.980 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:23.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:23 smithi082 bash[77063]: cluster 2023-10-09T04:54:22.846507+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:54:23.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:23 smithi082 bash[77063]: audit 2023-10-09T04:54:23.889340+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:23.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:23 smithi082 bash[79733]: cluster 2023-10-09T04:54:22.846507+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:54:23.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:23 smithi082 bash[79733]: audit 2023-10-09T04:54:23.889340+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:24.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:24.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:24.144 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:23 smithi153 bash[59565]: cluster 2023-10-09T04:54:22.846507+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1330: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:54:24.145 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:23 smithi153 bash[59565]: audit 2023-10-09T04:54:23.889340+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:24.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:24.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:24.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:24.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 38 16 59070 59054 97.1108 58.375 0.0265381 0.0102918 2023-10-09T04:54:24.398 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:24.432 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:24.443 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:24.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:24.577 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:24.683 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:24.797 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:54:24.822 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:24.918 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:24 smithi153 bash[59565]: audit 2023-10-09T04:54:23.901298+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:24.956 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:24.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:24 smithi082 bash[77063]: audit 2023-10-09T04:54:23.901298+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:24.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:24 smithi082 bash[79733]: audit 2023-10-09T04:54:23.901298+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:24.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:24.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:24.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:25.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:25.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:25.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:25.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:25.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:25.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:25.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:25.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 39 16 59992 59976 96.0983 57.625 0.00368188 0.0103771 2023-10-09T04:54:25.483 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:25.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:25.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:25.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:25.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:25.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:25.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:25.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:25.921 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:25.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:26.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:26.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:26.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:26.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:26 smithi153 bash[59565]: cluster 2023-10-09T04:54:24.848517+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:54:26.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:26 smithi153 bash[59565]: audit 2023-10-09T04:54:25.316430+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:26 smithi153 bash[59565]: audit 2023-10-09T04:54:25.322371+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:26.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:26.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:26.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:26.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:54:26.396592+0000 min lat: 0.00123875 max lat: 0.508003 avg lat: 0.0105051 2023-10-09T04:54:26.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:26.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 40 16 60918 60902 95.1426 57.875 0.00507421 0.0105051 2023-10-09T04:54:26.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:26.435 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[77063]: cluster 2023-10-09T04:54:24.848517+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:54:26.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[77063]: audit 2023-10-09T04:54:25.316430+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[77063]: audit 2023-10-09T04:54:25.322371+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[79733]: cluster 2023-10-09T04:54:24.848517+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1331: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:54:26.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[79733]: audit 2023-10-09T04:54:25.316430+0000 mon.a (mon.0) 1785 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:26 smithi082 bash[79733]: audit 2023-10-09T04:54:25.322371+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:26.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:26.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:26.554 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:26.583 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:26.686 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:26.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:26.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:26.842 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:26.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:26.986 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:26.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:26.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:27.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:27.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:27.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:27.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.245 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:27.353 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:27.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:27.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 41 16 62046 62030 94.5414 70.5 0.0155481 0.0105717 2023-10-09T04:54:27.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:27.501 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:27.509 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:27.612 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:27.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:27.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.901 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:27.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:28.004 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:28.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:28.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:28.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:28 smithi153 bash[59565]: cluster 2023-10-09T04:54:26.849102+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:54:28.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.278 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:28 smithi082 bash[79733]: cluster 2023-10-09T04:54:26.849102+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:54:28.279 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:28 smithi082 bash[77063]: cluster 2023-10-09T04:54:26.849102+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1332: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:54:28.298 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:28.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 42 16 63061 63045 93.8008 63.4375 0.00599508 0.0106534 2023-10-09T04:54:28.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:28.446 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:28.549 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:28.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:28.679 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:28.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:28.813 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:28.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:28.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:28.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:28.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:28.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:29.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:29.110 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:29.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:29.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:29.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:29.390 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:29.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 43 16 64189 64173 93.2588 70.5 0.00997181 0.0107167 2023-10-09T04:54:29.495 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:29.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:29.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:29.624 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:54:29.630 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:29.646 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:29.663 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:29.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:29.776 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:29] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-10-09T04:54:29.798 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:29.902 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:29.929 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:29.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:30.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:30.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:30.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:30.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:30.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:30.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:30.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 44 16 65232 65216 92.6207 65.1875 0.0080558 0.0107915 2023-10-09T04:54:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:30 smithi153 bash[59565]: cluster 2023-10-09T04:54:28.849843+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:30 smithi153 bash[59565]: audit 2023-10-09T04:54:29.298800+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:30.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:30.472 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:30 smithi082 bash[77063]: cluster 2023-10-09T04:54:28.849843+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:30 smithi082 bash[77063]: audit 2023-10-09T04:54:29.298800+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:30.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:30 smithi082 bash[79733]: cluster 2023-10-09T04:54:28.849843+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1333: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:30.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:30 smithi082 bash[79733]: audit 2023-10-09T04:54:29.298800+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:30.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:30.602 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:30.705 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:30.738 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:30.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:30.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:30.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:30.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:30.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:30.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:30.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:31.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:31.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:31.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:31.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:31.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:31.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 45 16 66296 66280 92.0401 66.5 0.00381511 0.0108596 2023-10-09T04:54:31.487 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:31.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:31.590 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:31.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:31.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:31.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:31.720 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:31.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:31.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:31.846 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:31.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:31.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:31.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:31.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:31.999 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:32.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:32.129 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:32.237 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:32.241 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:32 smithi153 bash[59565]: cluster 2023-10-09T04:54:30.850455+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2023-10-09T04:54:32.244 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:32 smithi082 bash[77063]: cluster 2023-10-09T04:54:30.850455+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2023-10-09T04:54:32.245 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:32 smithi082 bash[79733]: cluster 2023-10-09T04:54:30.850455+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1334: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2023-10-09T04:54:32.267 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:32.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:32.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.398 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 46 16 67253 67237 91.3395 59.8125 0.0198783 0.0109417 2023-10-09T04:54:32.399 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:32.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:32.634 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:32.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:32.803 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:32.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:32.933 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:32.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:33.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:33.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:33.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:33.298 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:33.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:33.397 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 47 15 67989 67974 90.3761 46.0625 0.0186356 0.0110591 2023-10-09T04:54:33.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:33.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.606 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:33.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:33.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:33.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:33.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:33.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:33.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:33.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:33.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:34.116 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:34.135 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:34 smithi082 bash[77063]: cluster 2023-10-09T04:54:32.851063+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:34.144 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:34.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:34.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:34.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:34.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:34.401 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 48 16 69092 69076 89.9224 68.875 0.018015 0.0111145 2023-10-09T04:54:34.403 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:34.407 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:34 smithi082 bash[79733]: cluster 2023-10-09T04:54:32.851063+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:34.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:34 smithi153 bash[59565]: cluster 2023-10-09T04:54:32.851063+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1335: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:54:34.509 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:34.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:34.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.668 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:34.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:34.769 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.797 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:34.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:34.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:34.851 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:54:34.851 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:54:34.851 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:54:34.851 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:54:34.851 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128681 9.13 9:c9643360:::benchmark_data_smithi082_200616_object19320:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.191082+0000", 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.65503465, 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3273182540000001, 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.852 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128681 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191082+0000", 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.853 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191082+0000", 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191080+0000", 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.022e-05 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.854 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191090+0000", 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3859999999999999e-05 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191104+0000", 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191113+0000", 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6236000000000007e-05 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191189+0000", 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3178e-05 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.191202+0000", 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.89141120100000004 2023-10-09T04:54:34.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.082614+0000", 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6563999999999999e-05 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.082630+0000", 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.404007502 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.486638+0000", 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00078466699999999996 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.487422+0000", 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0090927160000000007 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.496515+0000", 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012738100000000001 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.496642+0000", 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00038249600000000002 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.497025+0000", 2023-10-09T04:54:34.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0099682380000000008 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.506993+0000", 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0094992240000000006 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.516492+0000", 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6177000000000002e-05 2023-10-09T04:54:34.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.516539+0000", 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7756000000000001e-05 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.516596+0000", 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018041330000000001 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.518400+0000", 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3273182540000001 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128692 9.13 9:cd18ad64:::benchmark_data_smithi082_200616_object19331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.234171+0000", 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.61194543400001, 2023-10-09T04:54:34.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2858657770000002, 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128692 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234171+0000", 2023-10-09T04:54:34.863 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234171+0000", 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:34.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234170+0000", 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0280000000000006e-06 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234179+0000", 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9142000000000001e-05 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234198+0000", 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8390000000000002e-06 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234208+0000", 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000100398 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234308+0000", 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6498999999999999e-05 2023-10-09T04:54:34.867 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.234325+0000", 2023-10-09T04:54:34.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.84859445099999997 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.082919+0000", 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0574000000000001e-05 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.082930+0000", 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.406285386 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.489215+0000", 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00097104699999999995 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.490186+0000", 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014927974 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.505114+0000", 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7819999999999997e-05 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.505192+0000", 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031222500000000001 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.505504+0000", 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0085344419999999997 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.514039+0000", 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044324500000000001 2023-10-09T04:54:34.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.518471+0000", 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8461000000000002e-05 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.518530+0000", 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3903e-05 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.518603+0000", 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001433728 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.520037+0000", 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2858657770000002 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.875 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128754 9.13 9:cec1c53f:::benchmark_data_smithi082_200616_object19393:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.293324+0000", 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.55279244600001, 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2327266610000001, 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128754 2023-10-09T04:54:34.876 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293324+0000", 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293324+0000", 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293323+0000", 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8780000000000003e-06 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293329+0000", 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.217e-05 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293341+0000", 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4679999999999997e-06 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.293348+0000", 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022031210000000002 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.295551+0000", 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0097e-05 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.295561+0000", 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.78745951999999997 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083021+0000", 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0186e-05 2023-10-09T04:54:34.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083031+0000", 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4072042259999999 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.490235+0000", 2023-10-09T04:54:34.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00082259200000000003 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.491058+0000", 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.021152687 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.882 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.512210+0000", 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9604999999999994e-05 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.883 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.512290+0000", 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031276699999999998 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.512603+0000", 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0075026609999999999 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.520105+0000", 2023-10-09T04:54:34.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3582999999999998e-05 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.520159+0000", 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035599849999999999 2023-10-09T04:54:34.886 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.523719+0000", 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7490999999999997e-05 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.523756+0000", 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0022947129999999999 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.888 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.526051+0000", 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2327266610000001 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.889 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128756 9.13 9:cbc99339:::benchmark_data_smithi082_200616_object19395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.297136+0000", 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.54898123800001, 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2785646430000002, 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.890 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128756 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.891 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297136+0000", 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297136+0000", 2023-10-09T04:54:34.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297134+0000", 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7559999999999999e-06 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297142+0000", 2023-10-09T04:54:34.893 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0913999999999999e-05 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297153+0000", 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7270000000000004e-06 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297158+0000", 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6033999999999999e-05 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297215+0000", 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0079e-05 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.297225+0000", 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.78583335899999995 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083058+0000", 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1829999999999994e-06 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.897 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083066+0000", 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.437137661 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.520204+0000", 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00082415500000000003 2023-10-09T04:54:34.898 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.521028+0000", 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014163979 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.535192+0000", 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9029000000000006e-05 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.535261+0000", 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000296373 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.535557+0000", 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013435610000000001 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.548993+0000", 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.024109979 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.573103+0000", 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000112619 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.573216+0000", 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6360999999999996e-05 2023-10-09T04:54:34.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.573292+0000", 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002408374 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.575700+0000", 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2785646430000002 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.902 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128764 9.13 9:ce5a0312:::benchmark_data_smithi082_200616_object19403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.305914+0000", 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.540202446, 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.272061399, 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.903 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128764 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305914+0000", 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305914+0000", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305913+0000", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4599999999999997e-06 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305920+0000", 2023-10-09T04:54:34.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1892999999999999e-05 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305932+0000", 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4810000000000003e-06 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305939+0000", 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8804000000000001e-05 2023-10-09T04:54:34.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.305997+0000", 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3709999999999999e-05 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.306011+0000", 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.77710719100000003 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083118+0000", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3183999999999999e-05 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083131+0000", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4379398290000001 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.521071+0000", 2023-10-09T04:54:34.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00084512699999999999 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.521916+0000", 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.021795156 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.543712+0000", 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000140409 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.543852+0000", 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000437296 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.544289+0000", 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.021200165999999999 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.565489+0000", 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.010299489 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.575789+0000", 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9572000000000004e-05 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.575859+0000", 2023-10-09T04:54:34.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.4651000000000001e-05 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.575933+0000", 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020426369999999999 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.577976+0000", 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.272061399 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.912 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128769 9.13 9:cf932bed:::benchmark_data_smithi082_200616_object19408:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.312134+0000", 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.533983076, 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2684090650000002, 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128769 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312134+0000", 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.914 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312134+0000", 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312132+0000", 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5379999999999996e-06 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312137+0000", 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.319e-06 2023-10-09T04:54:34.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312143+0000", 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8650000000000003e-06 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312150+0000", 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5338000000000002e-05 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312235+0000", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0645999999999999e-05 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.312246+0000", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.77095132799999999 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083197+0000", 2023-10-09T04:54:34.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5079000000000001e-05 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083212+0000", 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4387464999999999 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.521959+0000", 2023-10-09T04:54:34.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00083620600000000004 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.522795+0000", 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.030964986 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.553760+0000", 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1928999999999994e-05 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.553842+0000", 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031483100000000001 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.920 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.554157+0000", 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.017116869999999999 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.571274+0000", 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0067506190000000002 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.578024+0000", 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9744000000000001e-05 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.578104+0000", 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011314799999999999 2023-10-09T04:54:34.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.578217+0000", 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0023255369999999999 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.580543+0000", 2023-10-09T04:54:34.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2684090650000002 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128801 9.13 9:cd804abf:::benchmark_data_smithi082_200616_object19440:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.344254+0000", 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.50186254799999, 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2383152559999999, 2023-10-09T04:54:34.924 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128801 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.925 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344254+0000", 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344254+0000", 2023-10-09T04:54:34.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344253+0000", 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.729e-06 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.927 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344258+0000", 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7870000000000004e-06 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344265+0000", 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8539999999999999e-06 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344269+0000", 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8955000000000004e-05 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344368+0000", 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9830000000000008e-06 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.344378+0000", 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.73888939099999995 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083267+0000", 2023-10-09T04:54:34.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1038e-05 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083278+0000", 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4395842619999999 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.522862+0000", 2023-10-09T04:54:34.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00083383399999999999 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.523696+0000", 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.039751342000000002 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.563447+0000", 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0596e-05 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.563518+0000", 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00028726400000000002 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.563805+0000", 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0093116310000000008 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.573117+0000", 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0074878310000000003 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.580605+0000", 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7061000000000003e-05 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.580642+0000", 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3022000000000002e-05 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.580695+0000", 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018747359999999999 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.582570+0000", 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2383152559999999 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128823 9.13 9:ca59b587:::benchmark_data_smithi082_200616_object19462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.365049+0000", 2023-10-09T04:54:34.937 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.48106756000001, 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2237441179999999, 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128823 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365049+0000", 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365049+0000", 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365048+0000", 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6109999999999996e-06 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365055+0000", 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0423999999999999e-05 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365086+0000", 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9169999999999995e-06 2023-10-09T04:54:34.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365095+0000", 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9040000000000003e-05 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365164+0000", 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0971999999999999e-05 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.941 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.365175+0000", 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.71814391600000005 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083319+0000", 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8460000000000002e-06 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083325+0000", 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4427988780000001 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.526124+0000", 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0037939549999999999 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.529918+0000", 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.041406816999999999 2023-10-09T04:54:34.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.571325+0000", 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8546000000000003e-05 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.571404+0000", 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031216799999999998 2023-10-09T04:54:34.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.571716+0000", 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0088461819999999993 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.580562+0000", 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020683310000000001 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.582630+0000", 2023-10-09T04:54:34.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0035059829999999998 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.586136+0000", 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014043000000000001 2023-10-09T04:54:34.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.586277+0000", 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002516772 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.948 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.588793+0000", 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2237441179999999 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.949 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128826 9.13 9:cdf0d4cb:::benchmark_data_smithi082_200616_object19465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.369899+0000", 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.476217408, 2023-10-09T04:54:34.950 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2590517549999998, 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128826 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.951 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.952 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.952 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.952 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369899+0000", 2023-10-09T04:54:34.952 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.952 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369899+0000", 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.953 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.954 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.954 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369899+0000", 2023-10-09T04:54:34.954 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4450000000000001e-06 2023-10-09T04:54:34.954 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.954 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369902+0000", 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0132999999999999e-05 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.955 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369912+0000", 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6970000000000001e-06 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.369917+0000", 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015591499999999999 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.956 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.370073+0000", 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5281000000000002e-05 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.370088+0000", 2023-10-09T04:54:34.957 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.71327719599999995 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083366+0000", 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0882e-05 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.958 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083376+0000", 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4992840940000001 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.582661+0000", 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001138404 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.959 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.583799+0000", 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.01626733 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.600066+0000", 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011646000000000001 2023-10-09T04:54:34.960 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.600183+0000", 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00038832100000000001 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.600571+0000", 2023-10-09T04:54:34.961 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.018853971000000001 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.619425+0000", 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3183999999999994e-05 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.619518+0000", 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0069410009999999996 2023-10-09T04:54:34.962 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.626459+0000", 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4283000000000002e-05 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.626513+0000", 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024377600000000002 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.963 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.628951+0000", 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2590517549999998 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128859 9.13 9:c994147a:::benchmark_data_smithi082_200616_object19498:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.403664+0000", 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.44245280800001, 2023-10-09T04:54:34.964 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2272958350000001, 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128859 2023-10-09T04:54:34.965 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403664+0000", 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.966 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403664+0000", 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.967 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403663+0000", 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7660000000000001e-06 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403670+0000", 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3106e-05 2023-10-09T04:54:34.968 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403683+0000", 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0934000000000001e-05 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403694+0000", 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2458000000000004e-05 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.969 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403756+0000", 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0967e-05 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.403767+0000", 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.67963884699999999 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.970 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083406+0000", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7079999999999996e-06 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083411+0000", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5004221879999999 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.583833+0000", 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0010878929999999999 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.584921+0000", 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.021678640999999998 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.606599+0000", 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5376000000000003e-05 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.972 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.606695+0000", 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000378704 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.607074+0000", 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.01235985 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.973 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.619433+0000", 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0095612519999999993 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.628995+0000", 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.1257999999999998e-05 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.974 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.629026+0000", 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.4787000000000002e-05 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.629081+0000", 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0018792329999999999 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.975 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.630960+0000", 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2272958350000001 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128907 9.13 9:cb843cae:::benchmark_data_smithi082_200616_object19546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.976 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.476428+0000", 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.36968889299999, 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.158796884, 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128907 2023-10-09T04:54:34.977 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476428+0000", 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476428+0000", 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:34.978 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476427+0000", 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.979 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476434+0000", 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3643e-05 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476448+0000", 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7569999999999993e-06 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.980 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476457+0000", 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1131000000000002e-05 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476508+0000", 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.7080000000000003e-06 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.981 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.476516+0000", 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.60692137899999998 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083438+0000", 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9699999999999998e-06 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.982 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083443+0000", 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5015192320000001 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.584962+0000", 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0011169179999999999 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.586079+0000", 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.031214091999999999 2023-10-09T04:54:34.983 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.617293+0000", 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012724500000000001 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.617420+0000", 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00043138699999999998 2023-10-09T04:54:34.984 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.617852+0000", 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.013138107 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.630990+0000", 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.0224000000000002e-05 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.985 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.631010+0000", 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016711790000000001 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.632681+0000", 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5822999999999997e-05 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.986 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.632737+0000", 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002487927 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.635225+0000", 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.158796884 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:34.987 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:128985 9.13 9:cc83ca8e:::benchmark_data_smithi082_200616_object19624:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:50:23.591398+0000", 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 251.25471864100001, 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.142553956, 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:34.988 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 128985 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591398+0000", 2023-10-09T04:54:34.989 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591398+0000", 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:34.990 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591397+0000", 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.493e-06 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.991 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591402+0000", 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1005000000000001e-05 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591413+0000", 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6730000000000003e-06 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.992 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591420+0000", 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1143999999999996e-05 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591481+0000", 2023-10-09T04:54:34.993 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9599999999999995e-06 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:23.591491+0000", 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.49204350899999999 2023-10-09T04:54:34.994 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083534+0000", 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2140000000000004e-06 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.995 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:24.083539+0000", 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5475041309999999 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.996 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.631044+0000", 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00073257699999999999 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for degraded object", 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.631776+0000", 2023-10-09T04:54:34.997 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.02788262 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.659659+0000", 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000104886 2023-10-09T04:54:34.998 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.659764+0000", 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00038271699999999997 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:34.999 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.660146+0000", 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.025237463000000002 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.000 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.685384+0000", 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.04601558 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.001 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.731399+0000", 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.7625e-05 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.731437+0000", 2023-10-09T04:54:35.002 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.6912999999999997e-05 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.731474+0000", 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0024781930000000001 2023-10-09T04:54:35.003 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:50:25.733952+0000", 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.142553956 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.004 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162015 9.e 9:7a3cb87a:::benchmark_data_smithi082_200616_object52654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.086260+0000", 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.759857286, 2023-10-09T04:54:35.005 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2287445369999999, 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162015 2023-10-09T04:54:35.006 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086260+0000", 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.007 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086260+0000", 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.008 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086259+0000", 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6949999999999997e-06 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086264+0000", 2023-10-09T04:54:35.009 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1362000000000001e-05 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086276+0000", 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.511e-06 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.010 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086282+0000", 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0837000000000001e-05 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.011 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086333+0000", 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000179977 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.012 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086513+0000", 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012995699999999999 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.086643+0000", 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.019468076000000001 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.106111+0000", 2023-10-09T04:54:35.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014221414 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.120332+0000", 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.020068596000000001 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.140401+0000", 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4109999999999996e-06 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.140409+0000", 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.98112785800000002 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121537+0000", 2023-10-09T04:54:35.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1378e-05 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121549+0000", 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1933466319999999 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.314895+0000", 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000108768 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.315004+0000", 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2287445369999999 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162019 9.c 9:3a95327e:::benchmark_data_smithi082_200616_object52658:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e279)", 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.121348+0000", 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.72476845700001, 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2525328980000001, 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "reached pg", 2023-10-09T04:54:35.020 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162019 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121348+0000", 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.022 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121348+0000", 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121347+0000", 2023-10-09T04:54:35.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6269999999999999e-06 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121352+0000", 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4569999999999994e-06 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121361+0000", 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7909999999999999e-06 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121368+0000", 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031592799999999999 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121684+0000", 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.8140000000000001e-06 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op must wait for map", 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.121692+0000", 2023-10-09T04:54:35.027 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014019291999999999 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.135711+0000", 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9669999999999996e-06 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.135721+0000", 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.98504871500000002 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.120770+0000", 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3525e-05 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.120783+0000", 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2176763559999999 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.338460+0000", 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8456999999999997e-05 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for missing object", 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.338548+0000", 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.034415355000000002 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.031 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.372964+0000", 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00091777099999999995 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373881+0000", 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2525328980000001 2023-10-09T04:54:35.032 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162037 9.c 9:3f1852be:::benchmark_data_smithi082_200616_object52676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.138628+0000", 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.70748894900001, 2023-10-09T04:54:35.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2708869959999998, 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162037 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138628+0000", 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138628+0000", 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138626+0000", 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9e-06 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138633+0000", 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1396999999999999e-05 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138645+0000", 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5309999999999998e-06 2023-10-09T04:54:35.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.138651+0000", 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015349719999999999 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.140186+0000", 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1844e-05 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.140198+0000", 2023-10-09T04:54:35.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.98079691499999999 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.120995+0000", 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3570000000000004e-06 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121004+0000", 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2178324229999999 2023-10-09T04:54:35.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.338837+0000", 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033961400000000002 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.342233+0000", 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00029475999999999998 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.342528+0000", 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.030415639000000001 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.372943+0000", 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00096683499999999998 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373910+0000", 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.6284000000000001e-05 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373936+0000", 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.035355882999999998 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409292+0000", 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9991000000000001e-05 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409362+0000", 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015274400000000001 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409515+0000", 2023-10-09T04:54:35.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2708869959999998 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162038 9.e 9:766ca3e3:::benchmark_data_smithi082_200616_object52677:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.143310+0000", 2023-10-09T04:54:35.044 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.70280732000001, 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1873897609999999, 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162038 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.045 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143310+0000", 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143310+0000", 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143309+0000", 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8979999999999998e-06 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143312+0000", 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.003e-06 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143318+0000", 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.3560000000000001e-06 2023-10-09T04:54:35.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.143322+0000", 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000796637 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.144118+0000", 2023-10-09T04:54:35.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0893000000000001e-05 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.144129+0000", 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.97744909099999999 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121578+0000", 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8869999999999997e-06 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121584+0000", 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.193450855 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.315035+0000", 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012666700000000001 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.315162+0000", 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00066321800000000003 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.315825+0000", 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.014061236 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.329886+0000", 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00049267400000000002 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330379+0000", 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6078000000000001e-05 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330435+0000", 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3266999999999997e-05 2023-10-09T04:54:35.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330488+0000", 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1913000000000001e-05 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330560+0000", 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013908599999999999 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330699+0000", 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1873897609999999 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162047 9.e 9:790f9d76:::benchmark_data_smithi082_200616_object52686:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.154014+0000", 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.69210331299999, 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1805844969999999, 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.055 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162047 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154014+0000", 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154014+0000", 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:35.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154012+0000", 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3409999999999997e-06 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154019+0000", 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7210999999999998e-05 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.059 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154037+0000", 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3459999999999994e-06 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154045+0000", 2023-10-09T04:54:35.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3309000000000001e-05 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154098+0000", 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6800000000000005e-06 2023-10-09T04:54:35.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.154108+0000", 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96749955799999998 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121608+0000", 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8950000000000003e-06 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121613+0000", 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2003174350000001 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.321931+0000", 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017165099999999999 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.322103+0000", 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000195312 2023-10-09T04:54:35.064 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.322298+0000", 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0084648980000000002 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330763+0000", 2023-10-09T04:54:35.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1703999999999998e-05 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.330814+0000", 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0036055750000000002 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.334420+0000", 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4341999999999999e-05 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.334434+0000", 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2464000000000004e-05 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.334497+0000", 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010123099999999999 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.334598+0000", 2023-10-09T04:54:35.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1805844969999999 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162050 9.c 9:3dfbb0c4:::benchmark_data_smithi082_200616_object52689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.069 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.158701+0000", 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.68741605, 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2510487929999998, 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.070 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162050 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158701+0000", 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158701+0000", 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158699+0000", 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2290000000000002e-06 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158705+0000", 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4089999999999992e-06 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158714+0000", 2023-10-09T04:54:35.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2719999999999997e-06 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.158719+0000", 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0015448 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.160264+0000", 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1021e-05 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.160275+0000", 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.96075253599999999 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121028+0000", 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121036+0000", 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2478732969999999 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.368910+0000", 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000147303 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.369057+0000", 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017765700000000001 2023-10-09T04:54:35.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.369235+0000", 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0046804519999999999 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373915+0000", 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4650000000000001e-05 2023-10-09T04:54:35.078 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373960+0000", 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011351722 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.385311+0000", 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.024249322 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409561+0000", 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3072000000000003e-05 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409614+0000", 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013585 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409750+0000", 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2510487929999998 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162056 9.c 9:3450d4c7:::benchmark_data_smithi082_200616_object52695:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.167486+0000", 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.67863088999999, 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2424944249999998, 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.081 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162056 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167486+0000", 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167486+0000", 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167484+0000", 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7530000000000004e-06 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.083 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167491+0000", 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6557000000000001e-05 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167508+0000", 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.558e-06 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167513+0000", 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9416000000000001e-05 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167583+0000", 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2717e-05 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.085 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.167595+0000", 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.95346390000000003 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121059+0000", 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7330000000000003e-06 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.086 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.121067+0000", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2529062469999999 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.373973+0000", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000120383 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.374094+0000", 2023-10-09T04:54:35.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018765700000000001 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.374281+0000", 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.011038631 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.088 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.385320+0000", 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5058000000000002e-05 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.385355+0000", 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0084722289999999995 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.393827+0000", 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.015962603999999998 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409790+0000", 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1498000000000001e-05 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409841+0000", 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013903799999999999 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.409980+0000", 2023-10-09T04:54:35.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2424944249999998 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.092 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35685.0:162063 9.c 9:3fd834a4:::benchmark_data_smithi082_200616_object52702:head [delete] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:51:13.181774+0000", 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 201.664342619, 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2284440879999998, 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35685", 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:54:35.093 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 162063 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181774+0000", 2023-10-09T04:54:35.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181774+0000", 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181772+0000", 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2447e-05 2023-10-09T04:54:35.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181785+0000", 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9417000000000001e-05 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181804+0000", 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6749999999999997e-06 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181814+0000", 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1958999999999993e-05 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181886+0000", 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3936e-05 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:13.181900+0000", 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.941268145 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.123168+0000", 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3539e-05 2023-10-09T04:54:35.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for peered", 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:14.123181+0000", 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2549311699999999 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.378112+0000", 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020488160000000002 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.380161+0000", 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012600799999999999 2023-10-09T04:54:35.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.380287+0000", 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0051056290000000004 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.385393+0000", 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0084797629999999995 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.393873+0000", 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031886039999999998 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:54:35.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.397061+0000", 2023-10-09T04:54:35.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.012959106 2023-10-09T04:54:35.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.410020+0000", 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0890000000000002e-05 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:54:35.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.410071+0000", 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000147033 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:51:15.410218+0000", 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2284440879999998 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.106 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:54:35.107 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:54:35.107 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:54:35.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:35.138 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:35.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:35.262 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:35.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.401 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 49 16 70057 70041 89.3181 60.3125 0.00469146 0.0111891 2023-10-09T04:54:35.402 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:35.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:35.532 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:35.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:35.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:35.776 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:35.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:35.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:36.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:36.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:36.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:36.162 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:54:36.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:36.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:54:36.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:36.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:36.330 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:36.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:36.401 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 50 16 70947 70931 88.6442 55.625 0.00681493 0.0112534 2023-10-09T04:54:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:36 smithi153 bash[59565]: cluster 2023-10-09T04:54:34.851633+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:54:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:36 smithi153 bash[59565]: audit 2023-10-09T04:54:35.597234+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.82:0/708171518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:36 smithi153 bash[59565]: audit 2023-10-09T04:54:35.597657+0000 mon.a (mon.0) 1788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:36.473 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[79733]: cluster 2023-10-09T04:54:34.851633+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:54:36.474 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[79733]: audit 2023-10-09T04:54:35.597234+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.82:0/708171518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.474 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[79733]: audit 2023-10-09T04:54:35.597657+0000 mon.a (mon.0) 1788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[77063]: cluster 2023-10-09T04:54:34.851633+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1336: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:54:36.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[77063]: audit 2023-10-09T04:54:35.597234+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.82:0/708171518' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:36 smithi082 bash[77063]: audit 2023-10-09T04:54:35.597657+0000 mon.a (mon.0) 1788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:54:36.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:36.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:36.635 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:36.739 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:36.765 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:36.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:36.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:36.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:37.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:37.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:37.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:37.160 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:37.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:37.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:37.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[79733]: cluster 2023-10-09T04:54:36.142671+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[79733]: audit 2023-10-09T04:54:36.145982+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:54:37.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[79733]: cluster 2023-10-09T04:54:36.146130+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:54:37.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[79733]: cluster 2023-10-09T04:54:36.149343+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[77063]: cluster 2023-10-09T04:54:36.142671+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[77063]: audit 2023-10-09T04:54:36.145982+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:54:37.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[77063]: cluster 2023-10-09T04:54:36.146130+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:54:37.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:37 smithi082 bash[77063]: cluster 2023-10-09T04:54:36.149343+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.397 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:37.401 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 51 16 71827 71811 87.9844 55 0.0184117 0.0113607 2023-10-09T04:54:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:37 smithi153 bash[59565]: cluster 2023-10-09T04:54:36.142671+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:37 smithi153 bash[59565]: audit 2023-10-09T04:54:36.145982+0000 mon.a (mon.0) 1789 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:54:37.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:37 smithi153 bash[59565]: cluster 2023-10-09T04:54:36.146130+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:54:37.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:37 smithi153 bash[59565]: cluster 2023-10-09T04:54:36.149343+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T04:54:37.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:37.533 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:37.556 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:37.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:37.685 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:37.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:37.823 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:37.927 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:37.964 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:37.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:37.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:38.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:38.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:38.176 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:38 smithi082 bash[79733]: cluster 2023-10-09T04:54:36.852152+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s 2023-10-09T04:54:38.200 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:38.228 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:38.332 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:38.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:38 smithi153 bash[59565]: cluster 2023-10-09T04:54:36.852152+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s 2023-10-09T04:54:38.357 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:38.401 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 52 16 72677 72661 87.3141 53.125 0.00858771 0.0114471 2023-10-09T04:54:38.462 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:38.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:38 smithi082 bash[77063]: cluster 2023-10-09T04:54:36.852152+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s 2023-10-09T04:54:38.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:38.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:38.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:38.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:38.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:38.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:38.857 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:38.882 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:38.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:38.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.009 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:39.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:39.151 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:39.257 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:39.282 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:39.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:39.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 53 16 73672 73656 86.8388 62.1875 0.0441259 0.0115087 2023-10-09T04:54:39.413 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:39.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:39.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:39.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:39.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:39.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:39.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:39.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:39.914 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:39] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.33.4" 2023-10-09T04:54:39.930 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:39.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:39.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:40.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:40.064 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:40.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:40.169 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:40.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:40.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:40.328 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:40.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 54 16 74693 74677 86.4123 63.8125 0.0104919 0.0115664 2023-10-09T04:54:40.431 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:40.439 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:40 smithi153 bash[59565]: cluster 2023-10-09T04:54:38.852774+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T04:54:40.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:40 smithi082 bash[77063]: cluster 2023-10-09T04:54:38.852774+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T04:54:40.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:40 smithi082 bash[79733]: cluster 2023-10-09T04:54:38.852774+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T04:54:40.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:40.578 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:40.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:40.680 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:40.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:40.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:40.849 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:40.954 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:40.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:40.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:41.013 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:54:41.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:41.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:41.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:41.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:41.382 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:41.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 55 16 75642 75626 85.9195 59.3125 0.00896381 0.0116321 2023-10-09T04:54:41.488 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:41.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:41.620 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:41.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:41.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:41.799 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:41.875 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:54:41.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:54:41.903 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:41.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:41.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:42.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:42.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:42.187 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:42.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:42.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:42.361 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:54:42.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:42.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 56 16 76555 76539 85.4042 57.0625 0.0127032 0.0117032 2023-10-09T04:54:42.405 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:54:42.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:42 smithi082 bash[79733]: cluster 2023-10-09T04:54:40.853361+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T04:54:42.463 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:54:42.465 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:42.477 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:42 smithi153 bash[59565]: cluster 2023-10-09T04:54:40.853361+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T04:54:42.491 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:42.509 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:54:42.546 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:54:42.582 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:54:42.594 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:42.621 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:54:42.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:42.654 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:54:42.658 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.658 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.658 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.659 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.659 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.659 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.659 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.659 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:54:42.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:42.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:42 smithi082 bash[77063]: cluster 2023-10-09T04:54:40.853361+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T04:54:42.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:42.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:42.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:42.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:42.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:43.003 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:43.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:43.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:43.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:43.318 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:43.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 57 16 77546 77530 84.9925 61.9375 0.00724402 0.0117596 2023-10-09T04:54:43.422 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:43.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:43 smithi153 bash[59565]: audit 2023-10-09T04:54:42.294101+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.82:0/3662366587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:43.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:43.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:43 smithi082 bash[77063]: audit 2023-10-09T04:54:42.294101+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.82:0/3662366587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:43.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:43 smithi082 bash[79733]: audit 2023-10-09T04:54:42.294101+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.82:0/3662366587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:43.556 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.579 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:43.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:43.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:43.839 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:43.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:43.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:44.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:44.095 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:44.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:44.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:44.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:44.360 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:44.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 58 16 78539 78523 84.5971 62.0625 0.017937 0.0118155 2023-10-09T04:54:44.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:44 smithi153 bash[59565]: cluster 2023-10-09T04:54:42.853980+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T04:54:44.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:44.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:44 smithi082 bash[77063]: cluster 2023-10-09T04:54:42.853980+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T04:54:44.474 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:44 smithi082 bash[79733]: cluster 2023-10-09T04:54:42.853980+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T04:54:44.492 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:44.597 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:44.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:44.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:44.749 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:44.857 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:44.898 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:44.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:45.005 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:45.028 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:45.131 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:45.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:45.235 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:45 smithi082 bash[77063]: audit 2023-10-09T04:54:44.298462+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:45.236 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:45 smithi082 bash[79733]: audit 2023-10-09T04:54:44.298462+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:45.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:45.274 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:45 smithi153 bash[59565]: audit 2023-10-09T04:54:44.298462+0000 mon.a (mon.0) 1791 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:45.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:45.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 59 16 79474 79458 84.1537 58.4375 0.0104467 0.0118769 2023-10-09T04:54:45.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:45.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.563 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:45.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:45.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:45.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:45.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:45.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:45.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:46.081 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:46.107 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:46.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:46.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:46.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:46.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:46.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:54:46.402219+0000 min lat: 0.00123875 max lat: 0.508003 avg lat: 0.0119454 2023-10-09T04:54:46.403 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:46.403 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 60 16 80355 80339 83.6688 55.0625 0.0501811 0.0119454 2023-10-09T04:54:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:46 smithi153 bash[59565]: cluster 2023-10-09T04:54:44.854526+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:46.431 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:54:46.431 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:54:46.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:46 smithi082 bash[77063]: cluster 2023-10-09T04:54:44.854526+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:46.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:46 smithi082 bash[79733]: cluster 2023-10-09T04:54:44.854526+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:46.490 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:46.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:46.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:46.648 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:46.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:46.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:46.890 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:46.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:46.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:47.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.050 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:47.159 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:47.184 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:47.240 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:54:47.288 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:47.296 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:47 smithi153 bash[59565]: audit 2023-10-09T04:54:46.644362+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.82:0/10488035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.296 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:47 smithi153 bash[59565]: audit 2023-10-09T04:54:46.644906+0000 mon.a (mon.0) 1792 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.296 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:47 smithi153 bash[59565]: cluster 2023-10-09T04:54:46.855131+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:54:47.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:47.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:47.402 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 61 16 81237 81221 83.2008 55.125 0.0154927 0.0120133 2023-10-09T04:54:47.416 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[77063]: audit 2023-10-09T04:54:46.644362+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.82:0/10488035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[77063]: audit 2023-10-09T04:54:46.644906+0000 mon.a (mon.0) 1792 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[77063]: cluster 2023-10-09T04:54:46.855131+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[79733]: audit 2023-10-09T04:54:46.644362+0000 mon.b (mon.1) 31 : audit 1 from='client.? 172.21.15.82:0/10488035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[79733]: audit 2023-10-09T04:54:46.644906+0000 mon.a (mon.0) 1792 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:47.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:47 smithi082 bash[79733]: cluster 2023-10-09T04:54:46.855131+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T04:54:47.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:47.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:54:47.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:47.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:47.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:47.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:54:47.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:47.732 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:47.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:47.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:47.977 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:48.089 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:48.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:48.235 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:48.340 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:48.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:48.403 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 62 16 82129 82113 82.758 55.75 0.0193652 0.0120774 2023-10-09T04:54:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.213669+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.227033+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.227108+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: audit 2023-10-09T04:54:47.230916+0000 mon.a (mon.0) 1794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.231006+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.236194+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.236797+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[79733]: cluster 2023-10-09T04:54:47.237939+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.213669+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.227033+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.227108+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: audit 2023-10-09T04:54:47.230916+0000 mon.a (mon.0) 1794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.231006+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:54:48.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.236194+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.236797+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:48 smithi082 bash[77063]: cluster 2023-10-09T04:54:47.237939+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:48.498 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.213669+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.227033+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.227108+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: audit 2023-10-09T04:54:47.230916+0000 mon.a (mon.0) 1794 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.231006+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.236194+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.236797+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:48 smithi153 bash[59565]: cluster 2023-10-09T04:54:47.237939+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T04:54:48.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:48.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:48.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:48.652 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:48.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:48.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:48.795 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:48.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:48.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:48.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:49.057 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:49.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:49.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:49.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:49.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:49.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:49.344 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:49 smithi153 bash[59565]: cluster 2023-10-09T04:54:48.858389+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:49.361 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:49.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:49.403 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 63 16 83110 83094 82.4176 61.3125 0.00757558 0.0121279 2023-10-09T04:54:49.452 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:49 smithi082 bash[77063]: cluster 2023-10-09T04:54:48.858389+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:49.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:49 smithi082 bash[79733]: cluster 2023-10-09T04:54:48.858389+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:49.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:49.504 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:49.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:49.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:49] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.33.4" 2023-10-09T04:54:49.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:49.883 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.913 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:49.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:50.017 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:50.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:50.157 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:50.184 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:50.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:50.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:50.403 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 64 16 84017 84001 82.0155 56.6875 0.00483696 0.0121863 2023-10-09T04:54:50.414 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:50.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:50.462 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:50.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:50.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:50.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:50.830 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:50.856 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:50.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:50.991 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:50.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:50.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:51.097 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.125 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:51.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:51.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[77063]: audit 2023-10-09T04:54:50.014749+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[77063]: audit 2023-10-09T04:54:50.028428+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[77063]: audit 2023-10-09T04:54:50.029896+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[77063]: audit 2023-10-09T04:54:50.031216+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[77063]: audit 2023-10-09T04:54:50.043537+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.377 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[79733]: audit 2023-10-09T04:54:50.014749+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.378 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[79733]: audit 2023-10-09T04:54:50.028428+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.378 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[79733]: audit 2023-10-09T04:54:50.029896+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:51.378 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[79733]: audit 2023-10-09T04:54:50.031216+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:51.378 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:51 smithi082 bash[79733]: audit 2023-10-09T04:54:50.043537+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.405 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 65 16 84949 84933 81.647 58.25 0.0185772 0.012241 2023-10-09T04:54:51.408 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:51.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:51 smithi153 bash[59565]: audit 2023-10-09T04:54:50.014749+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:51 smithi153 bash[59565]: audit 2023-10-09T04:54:50.028428+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:51 smithi153 bash[59565]: audit 2023-10-09T04:54:50.029896+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:51.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:51 smithi153 bash[59565]: audit 2023-10-09T04:54:50.031216+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:51.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:51 smithi153 bash[59565]: audit 2023-10-09T04:54:50.043537+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:54:51.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:51.540 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:51.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:51.647 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:51.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:51.800 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:51.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:54:51.908 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:51.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:51.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:51.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:52.055 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:52.083 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:52.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:52.197 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:52 smithi082 bash[77063]: cluster 2023-10-09T04:54:50.858831+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:52.197 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:52 smithi082 bash[79733]: cluster 2023-10-09T04:54:50.858831+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:52.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:52.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:52.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:52.359 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:54:52.386 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:52 smithi153 bash[59565]: cluster 2023-10-09T04:54:50.858831+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:54:52.405 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 66 16 85828 85812 81.2423 54.9375 0.0150373 0.0123035 2023-10-09T04:54:52.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:52.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:52.580 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:52.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:52.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:52.731 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:52.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:52.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:52.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:52.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:52.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:53.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:53.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:53.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:53.255 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:53.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:53.389 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:53.405 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 67 16 86764 86748 80.9028 58.5 0.00725943 0.0123547 2023-10-09T04:54:53.415 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:53.518 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:53.549 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:53.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:53.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:53.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:53.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:53.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:53.924 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:53.952 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:53.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:53.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:54.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:54.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:54.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:54.201 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:54.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:54.234 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:54 smithi082 bash[79733]: cluster 2023-10-09T04:54:52.859440+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:54:54.234 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:54 smithi082 bash[77063]: cluster 2023-10-09T04:54:52.859440+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:54:54.315 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:54.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:54.405 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 68 16 87672 87656 80.5476 56.75 0.018051 0.0124091 2023-10-09T04:54:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:54 smithi153 bash[59565]: cluster 2023-10-09T04:54:52.859440+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:54:54.449 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:54.476 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:54.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:54.617 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:54.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:54.747 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:54.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:54.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:54.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:54.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:55.008 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:55.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:55.113 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:55.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:55.246 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:55.273 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:55.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:55.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:55.405 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 69 16 88595 88579 80.2162 57.6875 0.0316768 0.0124607 2023-10-09T04:54:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:55.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:55.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:55.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.668 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:55.771 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:55.796 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:55.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:55.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:55.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:56.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:56.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:56.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:56.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:56 smithi153 bash[59565]: cluster 2023-10-09T04:54:54.860011+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:54:56.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.212 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:56.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:56.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:56.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:56.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:56.406 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 70 16 89516 89500 79.8925 57.5625 0.00854563 0.0125115 2023-10-09T04:54:56.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:56.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:56 smithi082 bash[77063]: cluster 2023-10-09T04:54:54.860011+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:54:56.453 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:56 smithi082 bash[79733]: cluster 2023-10-09T04:54:54.860011+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:54:56.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:56.575 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:56.579 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:56.596 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:56.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:56.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:56.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:56.841 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:56.871 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:56.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:56.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:57.110 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:57.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:57.248 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:57.274 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:57.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:57.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:57.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:57.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 71 16 90340 90324 79.4914 51.5 0.0065718 0.0125724 2023-10-09T04:54:57.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:57.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:57.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:57.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.647 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:54:57.647 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:54:57.661 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:54:57.663 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:54:57.663 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:57.766 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.794 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:57.896 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:57.921 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:57.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:58.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:58.064 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:58.171 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:58.179 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:58 smithi153 bash[59565]: cluster 2023-10-09T04:54:56.860586+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:54:58.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:58.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:58 smithi082 bash[77063]: cluster 2023-10-09T04:54:56.860586+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:54:58.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:58 smithi082 bash[79733]: cluster 2023-10-09T04:54:56.860586+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:54:58.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:58.325 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:58.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 72 15 91115 91100 79.0609 48.5 0.0148588 0.0126432 2023-10-09T04:54:58.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:58.463 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:58.567 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:58.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:58.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:58.737 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:58.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:58.867 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:58.903 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:54:58.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:58.994 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:58.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:59.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:59.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:59.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:59.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:59.377 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:59.386 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:54:59 smithi082 bash[77063]: audit 2023-10-09T04:54:57.645752+0000 mgr.y (mgr.34107) 1229 : audit 0 from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:54:59.386 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:54:59 smithi082 bash[79733]: audit 2023-10-09T04:54:57.645752+0000 mgr.y (mgr.34107) 1229 : audit 0 from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:54:59.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:59.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 73 16 91866 91850 78.62 46.875 0.0495622 0.0127123 2023-10-09T04:54:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:59.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:54:59 smithi153 bash[59565]: audit 2023-10-09T04:54:57.645752+0000 mgr.y (mgr.34107) 1229 : audit 0 from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:54:59.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:59.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:59.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:59.667 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:59.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:59.807 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:59.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:59.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:54:59.943 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:54:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:54:59] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-10-09T04:54:59.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:00.045 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:00.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:00.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:00.210 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:00.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:00 smithi082 bash[77063]: cluster 2023-10-09T04:54:58.861347+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:55:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:00 smithi082 bash[77063]: audit 2023-10-09T04:54:59.298680+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:00 smithi082 bash[79733]: cluster 2023-10-09T04:54:58.861347+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:55:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:00 smithi082 bash[79733]: audit 2023-10-09T04:54:59.298680+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:00.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:00.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:00.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:00.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 74 16 92636 92620 78.2079 48.125 0.00652571 0.0127807 2023-10-09T04:55:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:00 smithi153 bash[59565]: cluster 2023-10-09T04:54:58.861347+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:55:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:00 smithi153 bash[59565]: audit 2023-10-09T04:54:59.298680+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:00.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:00.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:00.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:00.589 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:00.699 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:00.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:00.827 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:00.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:00.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:00.984 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:00.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:01.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:01.113 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:01.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.240 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:01.295 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1349,"stamp":"2023-10-09T04:54:56.860217+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5710682035,"num_objects":87093,"num_object_clones":0,"num_object_copies":261279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87093,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":88171,"num_write_kb":5587319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":8398438,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88235,"ondisk_log_size":88235,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21973008,"kb_used_data":16666392,"kb_used_omap":47,"kb_used_meta":5306512,"kb_avail":634243056,"statfs":{"total":671965249536,"available":649464889344,"internally_reserved":0,"allocated":17066385408,"data_stored":17059945881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":5433868701},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":603586560,"num_objects":9210,"num_object_clones":0,"num_object_copies":27630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9210,"num_write_kb":589440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005524"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":39,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147803+0000","last_change":"2023-10-09T04:51:14.108186+0000","last_active":"2023-10-09T04:54:36.147803+0000","last_peered":"2023-10-09T04:54:36.147803+0000","last_clean":"2023-10-09T04:54:36.147803+0000","last_became_active":"2023-10-09T04:51:14.107642+0000","last_became_peered":"2023-10-09T04:51:14.107642+0000","last_unstale":"2023-10-09T04:54:36.147803+0000","last_undegraded":"2023-10-09T04:54:36.147803+0000","last_fullsized":"2023-10-09T04:54:36.147803+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:32:39.703389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":273,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247506+0000","last_change":"2023-10-09T04:51:14.111673+0000","last_active":"2023-10-09T04:54:47.247506+0000","last_peered":"2023-10-09T04:54:47.247506+0000","last_clean":"2023-10-09T04:54:47.247506+0000","last_became_active":"2023-10-09T04:51:14.111303+0000","last_became_peered":"2023-10-09T04:51:14.111303+0000","last_unstale":"2023-10-09T04:54:47.247506+0000","last_undegraded":"2023-10-09T04:54:47.247506+0000","last_fullsized":"2023-10-09T04:54:47.247506+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:38.607841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":270,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.145013+0000","last_change":"2023-10-09T04:51:14.150395+0000","last_active":"2023-10-09T04:54:36.145013+0000","last_peered":"2023-10-09T04:54:36.145013+0000","last_clean":"2023-10-09T04:54:36.145013+0000","last_became_active":"2023-10-09T04:51:14.149329+0000","last_became_peered":"2023-10-09T04:51:14.149329+0000","last_unstale":"2023-10-09T04:54:36.145013+0000","last_undegraded":"2023-10-09T04:54:36.145013+0000","last_fullsized":"2023-10-09T04:54:36.145013+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:50.627469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":286,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150308+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:54:36.150308+0000","last_peered":"2023-10-09T04:54:36.150308+0000","last_clean":"2023-10-09T04:54:36.150308+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:54:36.150308+0000","last_undegraded":"2023-10-09T04:54:36.150308+0000","last_fullsized":"2023-10-09T04:54:36.150308+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":288,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150408+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:54:36.150408+0000","last_peered":"2023-10-09T04:54:36.150408+0000","last_clean":"2023-10-09T04:54:36.150408+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:54:36.150408+0000","last_undegraded":"2023-10-09T04:54:36.150408+0000","last_fullsized":"2023-10-09T04:54:36.150408+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":261,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152397+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:54:36.152397+0000","last_peered":"2023-10-09T04:54:36.152397+0000","last_clean":"2023-10-09T04:54:36.152397+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:54:36.152397+0000","last_undegraded":"2023-10-09T04:54:36.152397+0000","last_fullsized":"2023-10-09T04:54:36.152397+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":280,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150699+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:54:36.150699+0000","last_peered":"2023-10-09T04:54:36.150699+0000","last_clean":"2023-10-09T04:54:36.150699+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:54:36.150699+0000","last_undegraded":"2023-10-09T04:54:36.150699+0000","last_fullsized":"2023-10-09T04:54:36.150699+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.144756+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:54:36.144756+0000","last_peered":"2023-10-09T04:54:36.144756+0000","last_clean":"2023-10-09T04:54:36.144756+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:54:36.144756+0000","last_undegraded":"2023-10-09T04:54:36.144756+0000","last_fullsized":"2023-10-09T04:54:36.144756+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147503+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:54:36.147503+0000","last_peered":"2023-10-09T04:54:36.147503+0000","last_clean":"2023-10-09T04:54:36.147503+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:54:36.147503+0000","last_undegraded":"2023-10-09T04:54:36.147503+0000","last_fullsized":"2023-10-09T04:54:36.147503+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151826+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:54:36.151826+0000","last_peered":"2023-10-09T04:54:36.151826+0000","last_clean":"2023-10-09T04:54:36.151826+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:54:36.151826+0000","last_undegraded":"2023-10-09T04:54:36.151826+0000","last_fullsized":"2023-10-09T04:54:36.151826+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":142,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152554+0000","last_change":"2023-10-09T04:51:14.339838+0000","last_active":"2023-10-09T04:54:36.152554+0000","last_peered":"2023-10-09T04:54:36.152554+0000","last_clean":"2023-10-09T04:54:36.152554+0000","last_became_active":"2023-10-09T04:51:14.150185+0000","last_became_peered":"2023-10-09T04:51:14.150185+0000","last_unstale":"2023-10-09T04:54:36.152554+0000","last_undegraded":"2023-10-09T04:54:36.152554+0000","last_fullsized":"2023-10-09T04:54:36.152554+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":249,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152672+0000","last_change":"2023-10-09T04:50:44.052710+0000","last_active":"2023-10-09T04:54:36.152672+0000","last_peered":"2023-10-09T04:54:36.152672+0000","last_clean":"2023-10-09T04:54:36.152672+0000","last_became_active":"2023-10-09T04:50:44.052312+0000","last_became_peered":"2023-10-09T04:50:44.052312+0000","last_unstale":"2023-10-09T04:54:36.152672+0000","last_undegraded":"2023-10-09T04:54:36.152672+0000","last_fullsized":"2023-10-09T04:54:36.152672+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":249,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152448+0000","last_change":"2023-10-09T04:51:14.162470+0000","last_active":"2023-10-09T04:54:36.152448+0000","last_peered":"2023-10-09T04:54:36.152448+0000","last_clean":"2023-10-09T04:54:36.152448+0000","last_became_active":"2023-10-09T04:51:14.129144+0000","last_became_peered":"2023-10-09T04:51:14.129144+0000","last_unstale":"2023-10-09T04:54:36.152448+0000","last_undegraded":"2023-10-09T04:54:36.152448+0000","last_fullsized":"2023-10-09T04:54:36.152448+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152887+0000","last_change":"2023-10-09T04:51:14.113752+0000","last_active":"2023-10-09T04:54:36.152887+0000","last_peered":"2023-10-09T04:54:36.152887+0000","last_clean":"2023-10-09T04:54:36.152887+0000","last_became_active":"2023-10-09T04:51:14.113290+0000","last_became_peered":"2023-10-09T04:51:14.113290+0000","last_unstale":"2023-10-09T04:54:36.152887+0000","last_undegraded":"2023-10-09T04:54:36.152887+0000","last_fullsized":"2023-10-09T04:54:36.152887+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":157,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152190+0000","last_change":"2023-10-09T04:51:14.179666+0000","last_active":"2023-10-09T04:54:36.152190+0000","last_peered":"2023-10-09T04:54:36.152190+0000","last_clean":"2023-10-09T04:54:36.152190+0000","last_became_active":"2023-10-09T04:51:14.130965+0000","last_became_peered":"2023-10-09T04:51:14.130965+0000","last_unstale":"2023-10-09T04:54:36.152190+0000","last_undegraded":"2023-10-09T04:54:36.152190+0000","last_fullsized":"2023-10-09T04:54:36.152190+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":208,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.153080+0000","last_change":"2023-10-09T04:50:44.064722+0000","last_active":"2023-10-09T04:54:36.153080+0000","last_peered":"2023-10-09T04:54:36.153080+0000","last_clean":"2023-10-09T04:54:36.153080+0000","last_became_active":"2023-10-09T04:50:44.064373+0000","last_became_peered":"2023-10-09T04:50:44.064373+0000","last_unstale":"2023-10-09T04:54:36.153080+0000","last_undegraded":"2023-10-09T04:54:36.153080+0000","last_fullsized":"2023-10-09T04:54:36.153080+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":283,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150980+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:54:36.150980+0000","last_peered":"2023-10-09T04:54:36.150980+0000","last_clean":"2023-10-09T04:54:36.150980+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:54:36.150980+0000","last_undegraded":"2023-10-09T04:54:36.150980+0000","last_fullsized":"2023-10-09T04:54:36.150980+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":265,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147593+0000","last_change":"2023-10-09T04:53:31.658083+0000","last_active":"2023-10-09T04:54:36.147593+0000","last_peered":"2023-10-09T04:54:36.147593+0000","last_clean":"2023-10-09T04:54:36.147593+0000","last_became_active":"2023-10-09T04:53:31.657030+0000","last_became_peered":"2023-10-09T04:53:31.657030+0000","last_unstale":"2023-10-09T04:54:36.147593+0000","last_undegraded":"2023-10-09T04:54:36.147593+0000","last_fullsized":"2023-10-09T04:54:36.147593+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":130,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152090+0000","last_change":"2023-10-09T04:51:14.129515+0000","last_active":"2023-10-09T04:54:36.152090+0000","last_peered":"2023-10-09T04:54:36.152090+0000","last_clean":"2023-10-09T04:54:36.152090+0000","last_became_active":"2023-10-09T04:51:14.128945+0000","last_became_peered":"2023-10-09T04:51:14.128945+0000","last_unstale":"2023-10-09T04:54:36.152090+0000","last_undegraded":"2023-10-09T04:54:36.152090+0000","last_fullsized":"2023-10-09T04:54:36.152090+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":283,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150493+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:54:36.150493+0000","last_peered":"2023-10-09T04:54:36.150493+0000","last_clean":"2023-10-09T04:54:36.150493+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:54:36.150493+0000","last_undegraded":"2023-10-09T04:54:36.150493+0000","last_fullsized":"2023-10-09T04:54:36.150493+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":270,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152324+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:54:36.152324+0000","last_peered":"2023-10-09T04:54:36.152324+0000","last_clean":"2023-10-09T04:54:36.152324+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:54:36.152324+0000","last_undegraded":"2023-10-09T04:54:36.152324+0000","last_fullsized":"2023-10-09T04:54:36.152324+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.148944+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:54:36.148944+0000","last_peered":"2023-10-09T04:54:36.148944+0000","last_clean":"2023-10-09T04:54:36.148944+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:54:36.148944+0000","last_undegraded":"2023-10-09T04:54:36.148944+0000","last_fullsized":"2023-10-09T04:54:36.148944+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"306'5502","reported_seq":5523,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:54.955480+0000","last_change":"2023-10-09T04:53:24.353425+0000","last_active":"2023-10-09T04:54:54.955480+0000","last_peered":"2023-10-09T04:54:54.955480+0000","last_clean":"2023-10-09T04:54:54.955480+0000","last_became_active":"2023-10-09T04:53:24.352877+0000","last_became_peered":"2023-10-09T04:53:24.352877+0000","last_unstale":"2023-10-09T04:54:54.955480+0000","last_undegraded":"2023-10-09T04:54:54.955480+0000","last_fullsized":"2023-10-09T04:54:54.955480+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360579072,"num_objects":5502,"num_object_clones":0,"num_object_copies":16506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5502,"num_write_kb":352128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":138,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152973+0000","last_change":"2023-10-09T04:51:14.384573+0000","last_active":"2023-10-09T04:54:36.152973+0000","last_peered":"2023-10-09T04:54:36.152973+0000","last_clean":"2023-10-09T04:54:36.152973+0000","last_became_active":"2023-10-09T04:51:14.155381+0000","last_became_peered":"2023-10-09T04:51:14.155381+0000","last_unstale":"2023-10-09T04:54:36.152973+0000","last_undegraded":"2023-10-09T04:54:36.152973+0000","last_fullsized":"2023-10-09T04:54:36.152973+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":238,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147213+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:54:36.147213+0000","last_peered":"2023-10-09T04:54:36.147213+0000","last_clean":"2023-10-09T04:54:36.147213+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:54:36.147213+0000","last_undegraded":"2023-10-09T04:54:36.147213+0000","last_fullsized":"2023-10-09T04:54:36.147213+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"306'5481","reported_seq":5502,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.613647+0000","last_change":"2023-10-09T04:53:24.139931+0000","last_active":"2023-10-09T04:54:53.613647+0000","last_peered":"2023-10-09T04:54:53.613647+0000","last_clean":"2023-10-09T04:54:53.613647+0000","last_became_active":"2023-10-09T04:53:24.139560+0000","last_became_peered":"2023-10-09T04:53:24.139560+0000","last_unstale":"2023-10-09T04:54:53.613647+0000","last_undegraded":"2023-10-09T04:54:53.613647+0000","last_fullsized":"2023-10-09T04:54:53.613647+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359202816,"num_objects":5481,"num_object_clones":0,"num_object_copies":16443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5481,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":151,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152837+0000","last_change":"2023-10-09T04:51:14.431351+0000","last_active":"2023-10-09T04:54:36.152837+0000","last_peered":"2023-10-09T04:54:36.152837+0000","last_clean":"2023-10-09T04:54:36.152837+0000","last_became_active":"2023-10-09T04:51:14.156447+0000","last_became_peered":"2023-10-09T04:51:14.156447+0000","last_unstale":"2023-10-09T04:54:36.152837+0000","last_undegraded":"2023-10-09T04:54:36.152837+0000","last_fullsized":"2023-10-09T04:54:36.152837+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.144843+0000","last_change":"2023-10-09T04:51:14.150212+0000","last_active":"2023-10-09T04:54:36.144843+0000","last_peered":"2023-10-09T04:54:36.144843+0000","last_clean":"2023-10-09T04:54:36.144843+0000","last_became_active":"2023-10-09T04:51:14.149425+0000","last_became_peered":"2023-10-09T04:51:14.149425+0000","last_unstale":"2023-10-09T04:54:36.144843+0000","last_undegraded":"2023-10-09T04:54:36.144843+0000","last_fullsized":"2023-10-09T04:54:36.144843+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.783749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"306'5393","reported_seq":5415,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.045015+0000","last_change":"2023-10-09T04:53:24.349405+0000","last_active":"2023-10-09T04:54:52.045015+0000","last_peered":"2023-10-09T04:54:52.045015+0000","last_clean":"2023-10-09T04:54:52.045015+0000","last_became_active":"2023-10-09T04:53:24.348702+0000","last_became_peered":"2023-10-09T04:53:24.348702+0000","last_unstale":"2023-10-09T04:54:52.045015+0000","last_undegraded":"2023-10-09T04:54:52.045015+0000","last_fullsized":"2023-10-09T04:54:52.045015+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:31.168505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353435648,"num_objects":5393,"num_object_clones":0,"num_object_copies":16179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5393,"num_write_kb":345152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":125,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152651+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:54:36.152651+0000","last_peered":"2023-10-09T04:54:36.152651+0000","last_clean":"2023-10-09T04:54:36.152651+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:54:36.152651+0000","last_undegraded":"2023-10-09T04:54:36.152651+0000","last_fullsized":"2023-10-09T04:54:36.152651+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":290,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152390+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:54:36.152390+0000","last_peered":"2023-10-09T04:54:36.152390+0000","last_clean":"2023-10-09T04:54:36.152390+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:54:36.152390+0000","last_undegraded":"2023-10-09T04:54:36.152390+0000","last_fullsized":"2023-10-09T04:54:36.152390+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"306'5435","reported_seq":5457,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.585853+0000","last_change":"2023-10-09T04:53:24.127013+0000","last_active":"2023-10-09T04:54:52.585853+0000","last_peered":"2023-10-09T04:54:52.585853+0000","last_clean":"2023-10-09T04:54:52.585853+0000","last_became_active":"2023-10-09T04:53:24.126651+0000","last_became_peered":"2023-10-09T04:53:24.126651+0000","last_unstale":"2023-10-09T04:54:52.585853+0000","last_undegraded":"2023-10-09T04:54:52.585853+0000","last_fullsized":"2023-10-09T04:54:52.585853+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5435,"log_dups_size":0,"ondisk_log_size":5435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:33.048421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356188160,"num_objects":5435,"num_object_clones":0,"num_object_copies":16305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5435,"num_write_kb":347840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":239,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147346+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:54:36.147346+0000","last_peered":"2023-10-09T04:54:36.147346+0000","last_clean":"2023-10-09T04:54:36.147346+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:54:36.147346+0000","last_undegraded":"2023-10-09T04:54:36.147346+0000","last_fullsized":"2023-10-09T04:54:36.147346+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":303,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.149829+0000","last_change":"2023-10-09T04:51:14.146161+0000","last_active":"2023-10-09T04:54:36.149829+0000","last_peered":"2023-10-09T04:54:36.149829+0000","last_clean":"2023-10-09T04:54:36.149829+0000","last_became_active":"2023-10-09T04:51:14.145388+0000","last_became_peered":"2023-10-09T04:51:14.145388+0000","last_unstale":"2023-10-09T04:54:36.149829+0000","last_undegraded":"2023-10-09T04:54:36.149829+0000","last_fullsized":"2023-10-09T04:54:36.149829+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:03.104400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"306'5320","reported_seq":5342,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.016820+0000","last_change":"2023-10-09T04:53:24.350004+0000","last_active":"2023-10-09T04:54:52.016820+0000","last_peered":"2023-10-09T04:54:52.016820+0000","last_clean":"2023-10-09T04:54:52.016820+0000","last_became_active":"2023-10-09T04:53:24.349539+0000","last_became_peered":"2023-10-09T04:53:24.349539+0000","last_unstale":"2023-10-09T04:54:52.016820+0000","last_undegraded":"2023-10-09T04:54:52.016820+0000","last_fullsized":"2023-10-09T04:54:52.016820+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5320,"log_dups_size":0,"ondisk_log_size":5320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:55.535031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348651520,"num_objects":5320,"num_object_clones":0,"num_object_copies":15960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5320,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":242,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.149928+0000","last_change":"2023-10-09T04:51:14.181439+0000","last_active":"2023-10-09T04:54:36.149928+0000","last_peered":"2023-10-09T04:54:36.149928+0000","last_clean":"2023-10-09T04:54:36.149928+0000","last_became_active":"2023-10-09T04:51:14.150700+0000","last_became_peered":"2023-10-09T04:51:14.150700+0000","last_unstale":"2023-10-09T04:54:36.149928+0000","last_undegraded":"2023-10-09T04:54:36.149928+0000","last_fullsized":"2023-10-09T04:54:36.149928+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:26.754849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152271+0000","last_change":"2023-10-09T04:51:14.151318+0000","last_active":"2023-10-09T04:54:36.152271+0000","last_peered":"2023-10-09T04:54:36.152271+0000","last_clean":"2023-10-09T04:54:36.152271+0000","last_became_active":"2023-10-09T04:51:14.150870+0000","last_became_peered":"2023-10-09T04:51:14.150870+0000","last_unstale":"2023-10-09T04:54:36.152271+0000","last_undegraded":"2023-10-09T04:54:36.152271+0000","last_fullsized":"2023-10-09T04:54:36.152271+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"306'5450","reported_seq":5472,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.539842+0000","last_change":"2023-10-09T04:53:24.136245+0000","last_active":"2023-10-09T04:54:52.539842+0000","last_peered":"2023-10-09T04:54:52.539842+0000","last_clean":"2023-10-09T04:54:52.539842+0000","last_became_active":"2023-10-09T04:53:24.135900+0000","last_became_peered":"2023-10-09T04:53:24.135900+0000","last_unstale":"2023-10-09T04:54:52.539842+0000","last_undegraded":"2023-10-09T04:54:52.539842+0000","last_fullsized":"2023-10-09T04:54:52.539842+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5450,"log_dups_size":0,"ondisk_log_size":5450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:27.074775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357171200,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5450,"num_write_kb":348800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":282,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152292+0000","last_change":"2023-10-09T04:51:14.149198+0000","last_active":"2023-10-09T04:54:36.152292+0000","last_peered":"2023-10-09T04:54:36.152292+0000","last_clean":"2023-10-09T04:54:36.152292+0000","last_became_active":"2023-10-09T04:51:14.148826+0000","last_became_peered":"2023-10-09T04:51:14.148826+0000","last_unstale":"2023-10-09T04:54:36.152292+0000","last_undegraded":"2023-10-09T04:54:36.152292+0000","last_fullsized":"2023-10-09T04:54:36.152292+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":189,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152250+0000","last_change":"2023-10-09T04:51:14.109636+0000","last_active":"2023-10-09T04:54:36.152250+0000","last_peered":"2023-10-09T04:54:36.152250+0000","last_clean":"2023-10-09T04:54:36.152250+0000","last_became_active":"2023-10-09T04:51:14.109258+0000","last_became_peered":"2023-10-09T04:51:14.109258+0000","last_unstale":"2023-10-09T04:54:36.152250+0000","last_undegraded":"2023-10-09T04:54:36.152250+0000","last_fullsized":"2023-10-09T04:54:36.152250+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.321575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"306'5442","reported_seq":5463,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.641095+0000","last_change":"2023-10-09T04:53:24.349881+0000","last_active":"2023-10-09T04:54:53.641095+0000","last_peered":"2023-10-09T04:54:53.641095+0000","last_clean":"2023-10-09T04:54:53.641095+0000","last_became_active":"2023-10-09T04:53:24.348953+0000","last_became_peered":"2023-10-09T04:53:24.348953+0000","last_unstale":"2023-10-09T04:54:53.641095+0000","last_undegraded":"2023-10-09T04:54:53.641095+0000","last_fullsized":"2023-10-09T04:54:53.641095+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":348288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":139,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147850+0000","last_change":"2023-10-09T04:51:14.125565+0000","last_active":"2023-10-09T04:54:36.147850+0000","last_peered":"2023-10-09T04:54:36.147850+0000","last_clean":"2023-10-09T04:54:36.147850+0000","last_became_active":"2023-10-09T04:51:14.125345+0000","last_became_peered":"2023-10-09T04:51:14.125345+0000","last_unstale":"2023-10-09T04:54:36.147850+0000","last_undegraded":"2023-10-09T04:54:36.147850+0000","last_fullsized":"2023-10-09T04:54:36.147850+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:14.172443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152147+0000","last_change":"2023-10-09T04:51:14.146838+0000","last_active":"2023-10-09T04:54:36.152147+0000","last_peered":"2023-10-09T04:54:36.152147+0000","last_clean":"2023-10-09T04:54:36.152147+0000","last_became_active":"2023-10-09T04:51:14.146457+0000","last_became_peered":"2023-10-09T04:51:14.146457+0000","last_unstale":"2023-10-09T04:54:36.152147+0000","last_undegraded":"2023-10-09T04:54:36.152147+0000","last_fullsized":"2023-10-09T04:54:36.152147+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"306'5676","reported_seq":5697,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:56.363330+0000","last_change":"2023-10-09T04:53:24.350320+0000","last_active":"2023-10-09T04:54:56.363330+0000","last_peered":"2023-10-09T04:54:56.363330+0000","last_clean":"2023-10-09T04:54:56.363330+0000","last_became_active":"2023-10-09T04:53:24.349443+0000","last_became_peered":"2023-10-09T04:53:24.349443+0000","last_unstale":"2023-10-09T04:54:56.363330+0000","last_undegraded":"2023-10-09T04:54:56.363330+0000","last_fullsized":"2023-10-09T04:54:56.363330+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371982336,"num_objects":5676,"num_object_clones":0,"num_object_copies":17028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5676,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":283,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.144581+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:54:36.144581+0000","last_peered":"2023-10-09T04:54:36.144581+0000","last_clean":"2023-10-09T04:54:36.144581+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:54:36.144581+0000","last_undegraded":"2023-10-09T04:54:36.144581+0000","last_fullsized":"2023-10-09T04:54:36.144581+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":36,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152610+0000","last_change":"2023-10-09T04:51:14.155214+0000","last_active":"2023-10-09T04:54:36.152610+0000","last_peered":"2023-10-09T04:54:36.152610+0000","last_clean":"2023-10-09T04:54:36.152610+0000","last_became_active":"2023-10-09T04:51:14.154876+0000","last_became_peered":"2023-10-09T04:51:14.154876+0000","last_unstale":"2023-10-09T04:54:36.152610+0000","last_undegraded":"2023-10-09T04:54:36.152610+0000","last_fullsized":"2023-10-09T04:54:36.152610+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.c","version":"306'5520","reported_seq":5541,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:56.361686+0000","last_change":"2023-10-09T04:53:24.135648+0000","last_active":"2023-10-09T04:54:56.361686+0000","last_peered":"2023-10-09T04:54:56.361686+0000","last_clean":"2023-10-09T04:54:56.361686+0000","last_became_active":"2023-10-09T04:53:24.135052+0000","last_became_peered":"2023-10-09T04:53:24.135052+0000","last_unstale":"2023-10-09T04:54:56.361686+0000","last_undegraded":"2023-10-09T04:54:56.361686+0000","last_fullsized":"2023-10-09T04:54:56.361686+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361758720,"num_objects":5520,"num_object_clones":0,"num_object_copies":16560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5520,"num_write_kb":353280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":470,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.153333+0000","last_change":"2023-10-09T04:50:44.050055+0000","last_active":"2023-10-09T04:54:36.153333+0000","last_peered":"2023-10-09T04:54:36.153333+0000","last_clean":"2023-10-09T04:54:36.153333+0000","last_became_active":"2023-10-09T04:50:44.049620+0000","last_became_peered":"2023-10-09T04:50:44.049620+0000","last_unstale":"2023-10-09T04:54:36.153333+0000","last_undegraded":"2023-10-09T04:54:36.153333+0000","last_fullsized":"2023-10-09T04:54:36.153333+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.943483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147544+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:54:36.147544+0000","last_peered":"2023-10-09T04:54:36.147544+0000","last_clean":"2023-10-09T04:54:36.147544+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:54:36.147544+0000","last_undegraded":"2023-10-09T04:54:36.147544+0000","last_fullsized":"2023-10-09T04:54:36.147544+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":240,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147348+0000","last_change":"2023-10-09T04:51:14.108030+0000","last_active":"2023-10-09T04:54:36.147348+0000","last_peered":"2023-10-09T04:54:36.147348+0000","last_clean":"2023-10-09T04:54:36.147348+0000","last_became_active":"2023-10-09T04:51:14.107660+0000","last_became_peered":"2023-10-09T04:51:14.107660+0000","last_unstale":"2023-10-09T04:54:36.147348+0000","last_undegraded":"2023-10-09T04:54:36.147348+0000","last_fullsized":"2023-10-09T04:54:36.147348+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"306'5475","reported_seq":5496,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.617210+0000","last_change":"2023-10-09T04:53:24.137716+0000","last_active":"2023-10-09T04:54:53.617210+0000","last_peered":"2023-10-09T04:54:53.617210+0000","last_clean":"2023-10-09T04:54:53.617210+0000","last_became_active":"2023-10-09T04:53:24.137204+0000","last_became_peered":"2023-10-09T04:53:24.137204+0000","last_unstale":"2023-10-09T04:54:53.617210+0000","last_undegraded":"2023-10-09T04:54:53.617210+0000","last_fullsized":"2023-10-09T04:54:53.617210+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":768,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147455+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:54:36.147455+0000","last_peered":"2023-10-09T04:54:36.147455+0000","last_clean":"2023-10-09T04:54:36.147455+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:54:36.147455+0000","last_undegraded":"2023-10-09T04:54:36.147455+0000","last_fullsized":"2023-10-09T04:54:36.147455+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":156,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150201+0000","last_change":"2023-10-09T04:51:14.112695+0000","last_active":"2023-10-09T04:54:36.150201+0000","last_peered":"2023-10-09T04:54:36.150201+0000","last_clean":"2023-10-09T04:54:36.150201+0000","last_became_active":"2023-10-09T04:51:14.112434+0000","last_became_peered":"2023-10-09T04:51:14.112434+0000","last_unstale":"2023-10-09T04:54:36.150201+0000","last_undegraded":"2023-10-09T04:54:36.150201+0000","last_fullsized":"2023-10-09T04:54:36.150201+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:10.034798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":251,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151625+0000","last_change":"2023-10-09T04:50:44.062081+0000","last_active":"2023-10-09T04:54:36.151625+0000","last_peered":"2023-10-09T04:54:36.151625+0000","last_clean":"2023-10-09T04:54:36.151625+0000","last_became_active":"2023-10-09T04:50:44.061730+0000","last_became_peered":"2023-10-09T04:50:44.061730+0000","last_unstale":"2023-10-09T04:54:36.151625+0000","last_undegraded":"2023-10-09T04:54:36.151625+0000","last_fullsized":"2023-10-09T04:54:36.151625+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:52.943977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"306'5465","reported_seq":5486,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:54.954908+0000","last_change":"2023-10-09T04:53:24.136930+0000","last_active":"2023-10-09T04:54:54.954908+0000","last_peered":"2023-10-09T04:54:54.954908+0000","last_clean":"2023-10-09T04:54:54.954908+0000","last_became_active":"2023-10-09T04:53:24.136546+0000","last_became_peered":"2023-10-09T04:53:24.136546+0000","last_unstale":"2023-10-09T04:54:54.954908+0000","last_undegraded":"2023-10-09T04:54:54.954908+0000","last_fullsized":"2023-10-09T04:54:54.954908+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":349760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1386,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147771+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:54:36.147771+0000","last_peered":"2023-10-09T04:54:36.147771+0000","last_clean":"2023-10-09T04:54:36.147771+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:54:36.147771+0000","last_undegraded":"2023-10-09T04:54:36.147771+0000","last_fullsized":"2023-10-09T04:54:36.147771+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":252,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.244446+0000","last_change":"2023-10-09T04:50:44.047494+0000","last_active":"2023-10-09T04:54:47.244446+0000","last_peered":"2023-10-09T04:54:47.244446+0000","last_clean":"2023-10-09T04:54:47.244446+0000","last_became_active":"2023-10-09T04:50:44.047111+0000","last_became_peered":"2023-10-09T04:50:44.047111+0000","last_unstale":"2023-10-09T04:54:47.244446+0000","last_undegraded":"2023-10-09T04:54:47.244446+0000","last_fullsized":"2023-10-09T04:54:47.244446+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:49.698697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":176,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147963+0000","last_change":"2023-10-09T04:50:44.051844+0000","last_active":"2023-10-09T04:54:36.147963+0000","last_peered":"2023-10-09T04:54:36.147963+0000","last_clean":"2023-10-09T04:54:36.147963+0000","last_became_active":"2023-10-09T04:50:44.051376+0000","last_became_peered":"2023-10-09T04:50:44.051376+0000","last_unstale":"2023-10-09T04:54:36.147963+0000","last_undegraded":"2023-10-09T04:54:36.147963+0000","last_fullsized":"2023-10-09T04:54:36.147963+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"306'5432","reported_seq":5453,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.610677+0000","last_change":"2023-10-09T04:53:24.352741+0000","last_active":"2023-10-09T04:54:53.610677+0000","last_peered":"2023-10-09T04:54:53.610677+0000","last_clean":"2023-10-09T04:54:53.610677+0000","last_became_active":"2023-10-09T04:53:24.351966+0000","last_became_peered":"2023-10-09T04:53:24.351966+0000","last_unstale":"2023-10-09T04:54:53.610677+0000","last_undegraded":"2023-10-09T04:54:53.610677+0000","last_fullsized":"2023-10-09T04:54:53.610677+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":831,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147776+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:54:36.147776+0000","last_peered":"2023-10-09T04:54:36.147776+0000","last_clean":"2023-10-09T04:54:36.147776+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:54:36.147776+0000","last_undegraded":"2023-10-09T04:54:36.147776+0000","last_fullsized":"2023-10-09T04:54:36.147776+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":527,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151858+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:54:36.151858+0000","last_peered":"2023-10-09T04:54:36.151858+0000","last_clean":"2023-10-09T04:54:36.151858+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:54:36.151858+0000","last_undegraded":"2023-10-09T04:54:36.151858+0000","last_fullsized":"2023-10-09T04:54:36.151858+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":224,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147221+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:54:36.147221+0000","last_peered":"2023-10-09T04:54:36.147221+0000","last_clean":"2023-10-09T04:54:36.147221+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:54:36.147221+0000","last_undegraded":"2023-10-09T04:54:36.147221+0000","last_fullsized":"2023-10-09T04:54:36.147221+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.153428+0000","last_change":"2023-10-09T04:51:14.125228+0000","last_active":"2023-10-09T04:54:36.153428+0000","last_peered":"2023-10-09T04:54:36.153428+0000","last_clean":"2023-10-09T04:54:36.153428+0000","last_became_active":"2023-10-09T04:51:14.124848+0000","last_became_peered":"2023-10-09T04:51:14.124848+0000","last_unstale":"2023-10-09T04:54:36.153428+0000","last_undegraded":"2023-10-09T04:54:36.153428+0000","last_fullsized":"2023-10-09T04:54:36.153428+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"306'5249","reported_seq":5271,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.049937+0000","last_change":"2023-10-09T04:53:24.136377+0000","last_active":"2023-10-09T04:54:52.049937+0000","last_peered":"2023-10-09T04:54:52.049937+0000","last_clean":"2023-10-09T04:54:52.049937+0000","last_became_active":"2023-10-09T04:53:24.136061+0000","last_became_peered":"2023-10-09T04:53:24.136061+0000","last_unstale":"2023-10-09T04:54:52.049937+0000","last_undegraded":"2023-10-09T04:54:52.049937+0000","last_fullsized":"2023-10-09T04:54:52.049937+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5249,"log_dups_size":0,"ondisk_log_size":5249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:09.217233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":693,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151824+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:54:36.151824+0000","last_peered":"2023-10-09T04:54:36.151824+0000","last_clean":"2023-10-09T04:54:36.151824+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:54:36.151824+0000","last_undegraded":"2023-10-09T04:54:36.151824+0000","last_fullsized":"2023-10-09T04:54:36.151824+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":323,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150468+0000","last_change":"2023-10-09T04:51:14.151803+0000","last_active":"2023-10-09T04:54:36.150468+0000","last_peered":"2023-10-09T04:54:36.150468+0000","last_clean":"2023-10-09T04:54:36.150468+0000","last_became_active":"2023-10-09T04:51:14.151334+0000","last_became_peered":"2023-10-09T04:51:14.151334+0000","last_unstale":"2023-10-09T04:54:36.150468+0000","last_undegraded":"2023-10-09T04:54:36.150468+0000","last_fullsized":"2023-10-09T04:54:36.150468+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:21.885966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248543+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:54:47.248543+0000","last_peered":"2023-10-09T04:54:47.248543+0000","last_clean":"2023-10-09T04:54:47.248543+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:54:47.248543+0000","last_undegraded":"2023-10-09T04:54:47.248543+0000","last_fullsized":"2023-10-09T04:54:47.248543+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"306'5399","reported_seq":5420,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.609491+0000","last_change":"2023-10-09T04:53:24.140748+0000","last_active":"2023-10-09T04:54:53.609491+0000","last_peered":"2023-10-09T04:54:53.609491+0000","last_clean":"2023-10-09T04:54:53.609491+0000","last_became_active":"2023-10-09T04:53:24.140325+0000","last_became_peered":"2023-10-09T04:53:24.140325+0000","last_unstale":"2023-10-09T04:54:53.609491+0000","last_undegraded":"2023-10-09T04:54:53.609491+0000","last_fullsized":"2023-10-09T04:54:53.609491+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5399,"log_dups_size":0,"ondisk_log_size":5399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353828864,"num_objects":5399,"num_object_clones":0,"num_object_copies":16197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":564,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152651+0000","last_change":"2023-10-09T04:53:05.183235+0000","last_active":"2023-10-09T04:54:36.152651+0000","last_peered":"2023-10-09T04:54:36.152651+0000","last_clean":"2023-10-09T04:54:36.152651+0000","last_became_active":"2023-10-09T04:53:05.182472+0000","last_became_peered":"2023-10-09T04:53:05.182472+0000","last_unstale":"2023-10-09T04:54:36.152651+0000","last_undegraded":"2023-10-09T04:54:36.152651+0000","last_fullsized":"2023-10-09T04:54:36.152651+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":251,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151712+0000","last_change":"2023-10-09T04:50:44.052455+0000","last_active":"2023-10-09T04:54:36.151712+0000","last_peered":"2023-10-09T04:54:36.151712+0000","last_clean":"2023-10-09T04:54:36.151712+0000","last_became_active":"2023-10-09T04:50:44.052136+0000","last_became_peered":"2023-10-09T04:50:44.052136+0000","last_unstale":"2023-10-09T04:54:36.151712+0000","last_undegraded":"2023-10-09T04:54:36.151712+0000","last_fullsized":"2023-10-09T04:54:36.151712+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:32.549357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150109+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:54:36.150109+0000","last_peered":"2023-10-09T04:54:36.150109+0000","last_clean":"2023-10-09T04:54:36.150109+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:54:36.150109+0000","last_undegraded":"2023-10-09T04:54:36.150109+0000","last_fullsized":"2023-10-09T04:54:36.150109+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"306'5310","reported_seq":5331,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.651840+0000","last_change":"2023-10-09T04:53:24.137672+0000","last_active":"2023-10-09T04:54:53.651840+0000","last_peered":"2023-10-09T04:54:53.651840+0000","last_clean":"2023-10-09T04:54:53.651840+0000","last_became_active":"2023-10-09T04:53:24.137340+0000","last_became_peered":"2023-10-09T04:53:24.137340+0000","last_unstale":"2023-10-09T04:54:53.651840+0000","last_undegraded":"2023-10-09T04:54:53.651840+0000","last_fullsized":"2023-10-09T04:54:53.651840+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347996160,"num_objects":5310,"num_object_clones":0,"num_object_copies":15930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5310,"num_write_kb":339840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":125,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152790+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:54:36.152790+0000","last_peered":"2023-10-09T04:54:36.152790+0000","last_clean":"2023-10-09T04:54:36.152790+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:54:36.152790+0000","last_undegraded":"2023-10-09T04:54:36.152790+0000","last_fullsized":"2023-10-09T04:54:36.152790+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":366,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152389+0000","last_change":"2023-10-09T04:51:14.350042+0000","last_active":"2023-10-09T04:54:36.152389+0000","last_peered":"2023-10-09T04:54:36.152389+0000","last_clean":"2023-10-09T04:54:36.152389+0000","last_became_active":"2023-10-09T04:51:14.157185+0000","last_became_peered":"2023-10-09T04:51:14.157185+0000","last_unstale":"2023-10-09T04:54:36.152389+0000","last_undegraded":"2023-10-09T04:54:36.152389+0000","last_fullsized":"2023-10-09T04:54:36.152389+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151763+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:54:36.151763+0000","last_peered":"2023-10-09T04:54:36.151763+0000","last_clean":"2023-10-09T04:54:36.151763+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:54:36.151763+0000","last_undegraded":"2023-10-09T04:54:36.151763+0000","last_fullsized":"2023-10-09T04:54:36.151763+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"306'5516","reported_seq":5537,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:53.554421+0000","last_change":"2023-10-09T04:53:24.140212+0000","last_active":"2023-10-09T04:54:53.554421+0000","last_peered":"2023-10-09T04:54:53.554421+0000","last_clean":"2023-10-09T04:54:53.554421+0000","last_became_active":"2023-10-09T04:53:24.139824+0000","last_became_peered":"2023-10-09T04:53:24.139824+0000","last_unstale":"2023-10-09T04:54:53.554421+0000","last_undegraded":"2023-10-09T04:54:53.554421+0000","last_fullsized":"2023-10-09T04:54:53.554421+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5516,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":386,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152523+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:54:36.152523+0000","last_peered":"2023-10-09T04:54:36.152523+0000","last_clean":"2023-10-09T04:54:36.152523+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:54:36.152523+0000","last_undegraded":"2023-10-09T04:54:36.152523+0000","last_fullsized":"2023-10-09T04:54:36.152523+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":284,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248641+0000","last_change":"2023-10-09T04:51:14.178205+0000","last_active":"2023-10-09T04:54:47.248641+0000","last_peered":"2023-10-09T04:54:47.248641+0000","last_clean":"2023-10-09T04:54:47.248641+0000","last_became_active":"2023-10-09T04:51:14.147035+0000","last_became_peered":"2023-10-09T04:51:14.147035+0000","last_unstale":"2023-10-09T04:54:47.248641+0000","last_undegraded":"2023-10-09T04:54:47.248641+0000","last_fullsized":"2023-10-09T04:54:47.248641+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:04.219571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":211,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.148290+0000","last_change":"2023-10-09T04:51:14.110047+0000","last_active":"2023-10-09T04:54:36.148290+0000","last_peered":"2023-10-09T04:54:36.148290+0000","last_clean":"2023-10-09T04:54:36.148290+0000","last_became_active":"2023-10-09T04:51:14.109658+0000","last_became_peered":"2023-10-09T04:51:14.109658+0000","last_unstale":"2023-10-09T04:54:36.148290+0000","last_undegraded":"2023-10-09T04:54:36.148290+0000","last_fullsized":"2023-10-09T04:54:36.148290+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":281,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247911+0000","last_change":"2023-10-09T04:51:14.148515+0000","last_active":"2023-10-09T04:54:47.247911+0000","last_peered":"2023-10-09T04:54:47.247911+0000","last_clean":"2023-10-09T04:54:47.247911+0000","last_became_active":"2023-10-09T04:51:14.147834+0000","last_became_peered":"2023-10-09T04:51:14.147834+0000","last_unstale":"2023-10-09T04:54:47.247911+0000","last_undegraded":"2023-10-09T04:54:47.247911+0000","last_fullsized":"2023-10-09T04:54:47.247911+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:48.693686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":253,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247113+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:54:47.247113+0000","last_peered":"2023-10-09T04:54:47.247113+0000","last_clean":"2023-10-09T04:54:47.247113+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:54:47.247113+0000","last_undegraded":"2023-10-09T04:54:47.247113+0000","last_fullsized":"2023-10-09T04:54:47.247113+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":244,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147585+0000","last_change":"2023-10-09T04:51:14.120655+0000","last_active":"2023-10-09T04:54:36.147585+0000","last_peered":"2023-10-09T04:54:36.147585+0000","last_clean":"2023-10-09T04:54:36.147585+0000","last_became_active":"2023-10-09T04:51:14.109662+0000","last_became_peered":"2023-10-09T04:51:14.109662+0000","last_unstale":"2023-10-09T04:54:36.147585+0000","last_undegraded":"2023-10-09T04:54:36.147585+0000","last_fullsized":"2023-10-09T04:54:36.147585+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152609+0000","last_change":"2023-10-09T04:51:14.152593+0000","last_active":"2023-10-09T04:54:36.152609+0000","last_peered":"2023-10-09T04:54:36.152609+0000","last_clean":"2023-10-09T04:54:36.152609+0000","last_became_active":"2023-10-09T04:51:14.152155+0000","last_became_peered":"2023-10-09T04:51:14.152155+0000","last_unstale":"2023-10-09T04:54:36.152609+0000","last_undegraded":"2023-10-09T04:54:36.152609+0000","last_fullsized":"2023-10-09T04:54:36.152609+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:51.059337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":237,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147467+0000","last_change":"2023-10-09T04:51:14.164283+0000","last_active":"2023-10-09T04:54:36.147467+0000","last_peered":"2023-10-09T04:54:36.147467+0000","last_clean":"2023-10-09T04:54:36.147467+0000","last_became_active":"2023-10-09T04:51:14.111363+0000","last_became_peered":"2023-10-09T04:51:14.111363+0000","last_unstale":"2023-10-09T04:54:36.147467+0000","last_undegraded":"2023-10-09T04:54:36.147467+0000","last_fullsized":"2023-10-09T04:54:36.147467+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":229,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150863+0000","last_change":"2023-10-09T04:51:14.150509+0000","last_active":"2023-10-09T04:54:36.150863+0000","last_peered":"2023-10-09T04:54:36.150863+0000","last_clean":"2023-10-09T04:54:36.150863+0000","last_became_active":"2023-10-09T04:51:14.150128+0000","last_became_peered":"2023-10-09T04:51:14.150128+0000","last_unstale":"2023-10-09T04:54:36.150863+0000","last_undegraded":"2023-10-09T04:54:36.150863+0000","last_fullsized":"2023-10-09T04:54:36.150863+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:17.197700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":207,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.151950+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:54:36.151950+0000","last_peered":"2023-10-09T04:54:36.151950+0000","last_clean":"2023-10-09T04:54:36.151950+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:54:36.151950+0000","last_undegraded":"2023-10-09T04:54:36.151950+0000","last_fullsized":"2023-10-09T04:54:36.151950+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.147211+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T04:54:36.147211+0000","last_peered":"2023-10-09T04:54:36.147211+0000","last_clean":"2023-10-09T04:54:36.147211+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T04:54:36.147211+0000","last_undegraded":"2023-10-09T04:54:36.147211+0000","last_fullsized":"2023-10-09T04:54:36.147211+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":301,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.150775+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:54:36.150775+0000","last_peered":"2023-10-09T04:54:36.150775+0000","last_clean":"2023-10-09T04:54:36.150775+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:54:36.150775+0000","last_undegraded":"2023-10-09T04:54:36.150775+0000","last_fullsized":"2023-10-09T04:54:36.150775+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-10-09T04:54:36.152202+0000","last_change":"2023-10-09T04:51:14.146277+0000","last_active":"2023-10-09T04:54:36.152202+0000","last_peered":"2023-10-09T04:54:36.152202+0000","last_clean":"2023-10-09T04:54:36.152202+0000","last_became_active":"2023-10-09T04:51:14.145974+0000","last_became_peered":"2023-10-09T04:51:14.145974+0000","last_unstale":"2023-10-09T04:54:36.152202+0000","last_undegraded":"2023-10-09T04:54:36.152202+0000","last_fullsized":"2023-10-09T04:54:36.152202+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5705891840,"num_objects":87065,"num_object_clones":0,"num_object_copies":261195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87065,"num_write_kb":5572160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17111777280,"data_stored":17111777280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87065,"ondisk_log_size":87065,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":9830,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430766,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2688604,"kb_used_data":1784980,"kb_used_omap":0,"kb_used_meta":903616,"kb_avail":91056548,"statfs":{"total":95995035648,"available":93241905152,"internally_reserved":0,"allocated":1827819520,"data_stored":1827215673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925302784},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:27 2023","interfaces":[{"interface":"back","average":{"1min":30.649000000000001,"5min":9.5790000000000006,"15min":4.8520000000000003},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":422.35000000000002,"5min":422.35000000000002,"15min":422.35000000000002},"last":0.38},{"interface":"front","average":{"1min":32.039000000000001,"5min":9.827,"15min":4.8259999999999996},"min":{"1min":0.38600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":423.267,"5min":423.267,"15min":423.267},"last":3.8290000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:53:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.7850000000000001,"15min":2.2120000000000002},"min":{"1min":0.624,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0099999999999998,"5min":57.668999999999997,"15min":58.816000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":2.7000000000000002,"15min":2.1440000000000001},"min":{"1min":0.58399999999999996,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.3819999999999999,"5min":58.046999999999997,"15min":58.840000000000003},"last":0.46800000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.6850000000000001,"15min":1.6519999999999999},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.746,"5min":13.138999999999999,"15min":13.138999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.3260000000000001,"15min":1.944},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.7789999999999999,"5min":7.4530000000000003,"15min":11.583},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:53:56 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.8120000000000001,"15min":2.1110000000000002},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.26700000000000002},"max":{"1min":1.2549999999999999,"5min":8.6449999999999996,"15min":102.02800000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.9510000000000001,"15min":2.0830000000000002},"min":{"1min":0.624,"5min":0.34499999999999997,"15min":0.32100000000000001},"max":{"1min":2.032,"5min":13.195,"15min":101.973},"last":0.90300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:53:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.4410000000000001,"15min":1.3009999999999999},"min":{"1min":0.57899999999999996,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.1859999999999999,"5min":10.199,"15min":11.933999999999999},"last":0.504},{"interface":"front","average":{"1min":0.876,"5min":1.4690000000000001,"15min":1.3360000000000001},"min":{"1min":0.48099999999999998,"5min":0.309,"15min":0.308},"max":{"1min":1.337,"5min":8.1080000000000005,"15min":8.452},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.7390000000000001,"15min":1.496},"min":{"1min":0.41399999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.397,"5min":8.6050000000000004,"15min":11.268000000000001},"last":0.747},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.9119999999999999,"15min":1.6279999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":2.4300000000000002,"5min":9.1449999999999996,"15min":15.138999999999999},"last":0.96999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.173,"5min":1.6899999999999999,"15min":1.472},"min":{"1min":0.442,"5min":0.40400000000000003,"15min":0.36899999999999999},"max":{"1min":27.059000000000001,"5min":27.059000000000001,"15min":27.059000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.77,"15min":1.552},"min":{"1min":0.44,"5min":0.39300000000000002,"15min":0.36599999999999999},"max":{"1min":27.698,"5min":27.698,"15min":27.698},"last":0.56499999999999995}]}]},{"osd":6,"up_from":146,"seq":627065225466,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3061464,"kb_used_data":2438928,"kb_used_omap":5,"kb_used_meta":622522,"kb_avail":90683688,"statfs":{"total":95995035648,"available":92860096512,"internally_reserved":0,"allocated":2497462272,"data_stored":2496862428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":637463136},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.0569999999999999,"15min":3.1840000000000002},"min":{"1min":0.51200000000000001,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":3.2989999999999999,"5min":39.142000000000003,"15min":214.82300000000001},"last":0.311},{"interface":"front","average":{"1min":0.79800000000000004,"5min":1.4350000000000001,"15min":3.0070000000000001},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.254},"max":{"1min":1.256,"5min":36.988999999999997,"15min":213.34700000000001},"last":0.432}]},{"osd":1,"last update":"Mon Oct 9 04:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":2.6859999999999999,"15min":2.677},"min":{"1min":0.41199999999999998,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":3.448,"5min":40.709000000000003,"15min":76.081000000000003},"last":4.6779999999999999},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.3849999999999998,"15min":2.3359999999999999},"min":{"1min":0.38300000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.2000000000000002,"5min":40.414000000000001,"15min":75.546000000000006},"last":0.495}]},{"osd":2,"last update":"Mon Oct 9 04:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.119,"15min":1.629},"min":{"1min":0.35099999999999998,"5min":0.34000000000000002,"15min":0.32000000000000001},"max":{"1min":3.5190000000000001,"5min":6.8940000000000001,"15min":6.8940000000000001},"last":3.5190000000000001},{"interface":"front","average":{"1min":0.877,"5min":0.97899999999999998,"15min":1.3},"min":{"1min":0.38400000000000001,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":1.397,"5min":6.423,"15min":9.4290000000000003},"last":0.32500000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:53:53 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.425,"15min":1.526},"min":{"1min":0.54700000000000004,"5min":0.36899999999999999,"15min":0.29899999999999999},"max":{"1min":2.8130000000000002,"5min":7.476,"15min":9.8810000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.4530000000000001,"15min":1.47},"min":{"1min":0.51300000000000001,"5min":0.34499999999999997,"15min":0.32600000000000001},"max":{"1min":2.1720000000000002,"5min":8.9079999999999995,"15min":9.3190000000000008},"last":2.1360000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:53:53 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.84399999999999997,"15min":1.1339999999999999},"min":{"1min":0.48299999999999998,"5min":0.314,"15min":0.248},"max":{"1min":1.4430000000000001,"5min":4.0380000000000003,"15min":13.699999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.232,"15min":1.351},"min":{"1min":0.59599999999999997,"5min":0.35899999999999999,"15min":0.307},"max":{"1min":2.738,"5min":10.428000000000001,"15min":10.428000000000001},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.125,"15min":1.2929999999999999},"min":{"1min":0.42299999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":1.7889999999999999,"5min":10.871,"15min":10.871},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.984,"15min":1.675},"min":{"1min":0.44800000000000001,"5min":0.40200000000000002,"15min":0.29599999999999999},"max":{"1min":2.778,"5min":11.321999999999999,"15min":11.321999999999999},"last":0.75}]},{"osd":7,"last update":"Mon Oct 9 04:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.7090000000000001,"15min":1.5880000000000001},"min":{"1min":0.45800000000000002,"5min":0.38900000000000001,"15min":0.33000000000000002},"max":{"1min":2.617,"5min":10.701000000000001,"15min":16.527999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.089,"5min":1.4950000000000001,"15min":1.5149999999999999},"min":{"1min":0.68799999999999994,"5min":0.39400000000000002,"15min":0.33300000000000002},"max":{"1min":1.6910000000000001,"5min":6.6180000000000003,"15min":15.366},"last":0.48499999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958915,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2935156,"kb_used_data":2093996,"kb_used_omap":6,"kb_used_meta":841145,"kb_avail":90809996,"statfs":{"total":95995035648,"available":92989435904,"internally_reserved":0,"allocated":2144251904,"data_stored":2143103062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":861332492},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.2389999999999999,"15min":2.117},"min":{"1min":0.22800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":47.274000000000001,"5min":47.274000000000001,"15min":47.274000000000001},"last":5.5209999999999999},{"interface":"front","average":{"1min":4.3600000000000003,"5min":1.8939999999999999,"15min":2.04},"min":{"1min":0.36799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":47.372999999999998,"5min":47.372999999999998,"15min":47.372999999999998},"last":5.0869999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.1359999999999999,"15min":1.98},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.8799999999999999,"5min":8.2650000000000006,"15min":15.336},"last":1.9750000000000001},{"interface":"front","average":{"1min":0.746,"5min":1.1419999999999999,"15min":1.8240000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.221},"max":{"1min":2.6640000000000001,"5min":7.7229999999999999,"15min":13.444000000000001},"last":1.911}]},{"osd":3,"last update":"Mon Oct 9 04:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.99,"15min":1.7869999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.20999999999999999},"max":{"1min":1.802,"5min":40.210000000000001,"15min":40.210000000000001},"last":5.423},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.8620000000000001,"15min":1.766},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.20200000000000001},"max":{"1min":1.8340000000000001,"5min":40.127000000000002,"15min":40.127000000000002},"last":1.9370000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.387,"15min":1.458},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":4.1769999999999996,"5min":7.923,"15min":13.362},"last":2.1749999999999998},{"interface":"front","average":{"1min":1.752,"5min":1.627,"15min":1.599},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.316},"max":{"1min":9.6259999999999994,"5min":9.827,"15min":10.923999999999999},"last":5.468}]},{"osd":5,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.4410000000000001,"15min":1.635},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.4500000000000002,"5min":9.2910000000000004,"15min":14.494999999999999},"last":2.02},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.71,"15min":1.6559999999999999},"min":{"1min":0.41899999999999998,"5min":0.38700000000000001,"15min":0.32800000000000001},"max":{"1min":2.3849999999999998,"5min":7.9569999999999999,"15min":13.413},"last":2.1440000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2810000000000001,"5min":1.7549999999999999,"15min":1.7649999999999999},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.32600000000000001},"max":{"1min":18.295000000000002,"5min":18.295000000000002,"15min":18.295000000000002},"last":2.0680000000000001},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.7869999999999999,"15min":1.841},"min":{"1min":0.48699999999999999,"5min":0.41999999999999998,"15min":0.40500000000000003},"max":{"1min":18.34,"5min":18.34,"15min":18.34},"last":2.0960000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.5489999999999999,"15min":1.673},"min":{"1min":0.40699999999999997,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":4.1479999999999997,"5min":7.8899999999999997,"15min":13.474},"last":2.1139999999999999},{"interface":"front","average":{"1min":1.623,"5min":1.889,"15min":1.7909999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.36399999999999999},"max":{"1min":9.5909999999999993,"5min":9.8079999999999998,"15min":18.074000000000002},"last":5.4960000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122440,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4246588,"kb_used_data":3395124,"kb_used_omap":9,"kb_used_meta":851446,"kb_avail":89498564,"statfs":{"total":95995035648,"available":91646529536,"internally_reserved":0,"allocated":3476606976,"data_stored":3475454528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":871881201},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:54:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.9279999999999999,"15min":1.605},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":0.78600000000000003,"5min":62.396000000000001,"15min":62.396000000000001},"last":3.9409999999999998},{"interface":"front","average":{"1min":0.872,"5min":2.4049999999999998,"15min":2.0569999999999999},"min":{"1min":0.28100000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.2999999999999998,"5min":62.496000000000002,"15min":62.496000000000002},"last":0.27200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:54:05 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.88700000000000001,"15min":1.7989999999999999},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9169999999999998,"5min":5.1180000000000003,"15min":17.109000000000002},"last":3.9100000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.274,"15min":2.1419999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25600000000000001},"max":{"1min":3.266,"5min":6.3929999999999998,"15min":12.57},"last":4.0620000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:54:05 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.274,"15min":1.506},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.911,"5min":7.9059999999999997,"15min":18.303999999999998},"last":4.0750000000000002},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.7669999999999999,"15min":1.883},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":8.6020000000000003,"15min":14.956},"last":0.30599999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.702,"15min":1.6479999999999999},"min":{"1min":0.56000000000000005,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":7.181,"5min":9.6509999999999998,"15min":16.84},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.149,"5min":1.157,"15min":1.3320000000000001},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6429999999999998,"5min":5.048,"15min":39.957999999999998},"last":4.0049999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.597,"15min":1.6379999999999999},"min":{"1min":0.59199999999999997,"5min":0.38400000000000001,"15min":0.35399999999999998},"max":{"1min":4.117,"5min":6.3460000000000001,"15min":11.617000000000001},"last":3.9689999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.409,"15min":1.4359999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.7280000000000002,"5min":9.4900000000000002,"15min":10.244},"last":4.0449999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.157,"15min":1.3779999999999999},"min":{"1min":0.45900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.9320000000000004,"5min":6.9320000000000004,"15min":10.345000000000001},"last":4.0990000000000002},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.6519999999999999,"15min":1.7529999999999999},"min":{"1min":0.48699999999999999,"5min":0.41199999999999998,"15min":0.39400000000000002},"max":{"1min":7.7009999999999996,"5min":7.7009999999999996,"15min":11.57},"last":4.0880000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":2.1379999999999999,"15min":1.9379999999999999},"min":{"1min":0.69499999999999995,"5min":0.41699999999999998,"15min":0.38400000000000001},"max":{"1min":3.6549999999999998,"5min":43.276000000000003,"15min":43.276000000000003},"last":4.0289999999999999},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.476,"15min":1.9630000000000001},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.2469999999999999,"5min":46.064,"15min":46.064},"last":3.3170000000000002}]}]},{"osd":2,"up_from":234,"seq":1005022347385,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2309980,"kb_used_data":2048468,"kb_used_omap":13,"kb_used_meta":261490,"kb_avail":91435172,"statfs":{"total":95995035648,"available":93629616128,"internally_reserved":0,"allocated":2097631232,"data_stored":2096450018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":267766655},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":3.371,"15min":2.8719999999999999},"min":{"1min":0.39900000000000002,"5min":0.186,"15min":0.186},"max":{"1min":3.895,"5min":112.312,"15min":112.312},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.5009999999999999,"5min":3.593,"15min":2.8929999999999998},"min":{"1min":0.22600000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":11.792,"5min":117.881,"15min":117.881},"last":0.66100000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":9.6449999999999996,"15min":4.7679999999999998},"min":{"1min":0.29299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":2.4980000000000002,"5min":267.54000000000002,"15min":267.54000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.0529999999999999,"5min":9.8040000000000003,"15min":4.4420000000000002},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6420000000000003,"5min":268.12200000000001,"15min":268.12200000000001},"last":2.7360000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.1499999999999999,"15min":1.4510000000000001},"min":{"1min":0.28499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.0199999999999996,"5min":14.334,"15min":14.340999999999999},"last":2.77},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.3140000000000001,"15min":2.0840000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.847,"5min":13.167999999999999,"15min":13.167999999999999},"last":0.98499999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.2190000000000001,"15min":1.7230000000000001},"min":{"1min":0.71399999999999997,"5min":0.35399999999999998,"15min":0.34599999999999997},"max":{"1min":3.9340000000000002,"5min":10.051,"15min":10.051},"last":0.498},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.96799999999999997,"15min":1.5940000000000001},"min":{"1min":0.40899999999999997,"5min":0.316,"15min":0.316},"max":{"1min":2.2869999999999999,"5min":10.119999999999999,"15min":10.119999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.254,"15min":1.7330000000000001},"min":{"1min":0.53900000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.6890000000000001,"5min":10.856999999999999,"15min":10.856999999999999},"last":2.6829999999999998},{"interface":"front","average":{"1min":1.238,"5min":1.373,"15min":2.2130000000000001},"min":{"1min":0.62,"5min":0.316,"15min":0.316},"max":{"1min":3.9750000000000001,"5min":11.75,"15min":12.295999999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.157,"15min":1.776},"min":{"1min":0.42299999999999999,"5min":0.371,"15min":0.371},"max":{"1min":2.2240000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":3.3500000000000001},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.159,"15min":1.4770000000000001},"min":{"1min":0.61899999999999999,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":6.7329999999999997,"5min":10.890000000000001,"15min":10.890000000000001},"last":2.6389999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":1.575,"15min":1.984},"min":{"1min":0.54900000000000004,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":5.3460000000000001,"5min":10.007999999999999,"15min":14.161},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.2350000000000001,"15min":1.708},"min":{"1min":0.47499999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":2.3210000000000002,"5min":9.8550000000000004,"15min":10.996},"last":0.62}]}]},{"osd":3,"up_from":103,"seq":442381631861,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3668940,"kb_used_data":2798468,"kb_used_omap":5,"kb_used_meta":870458,"kb_avail":90076212,"statfs":{"total":95995035648,"available":92238041088,"internally_reserved":0,"allocated":2865631232,"data_stored":2864462337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":891349166},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":4.8719999999999999,"15min":3.9489999999999998},"min":{"1min":0.33400000000000002,"5min":0.21199999999999999,"15min":0.19500000000000001},"max":{"1min":3.823,"5min":144.625,"15min":341.28699999999998},"last":4.9400000000000004},{"interface":"front","average":{"1min":1.335,"5min":5.1639999999999997,"15min":3.9809999999999999},"min":{"1min":0.442,"5min":0.23200000000000001,"15min":0.20000000000000001},"max":{"1min":8.766,"5min":144.24100000000001,"15min":341.57799999999997},"last":2.3559999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":4.1299999999999999,"15min":2.661},"min":{"1min":0.25800000000000001,"5min":0.21199999999999999,"15min":0.20100000000000001},"max":{"1min":3.4529999999999998,"5min":143.34100000000001,"15min":143.34100000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.69199999999999995,"5min":4.2400000000000002,"15min":2.8959999999999999},"min":{"1min":0.372,"5min":0.20399999999999999,"15min":0.188},"max":{"1min":3.2799999999999998,"5min":149.15700000000001,"15min":149.15700000000001},"last":4.9720000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.3149999999999999,"15min":1.4830000000000001},"min":{"1min":0.433,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":5.7069999999999999,"5min":10.148,"15min":10.148},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.2909999999999999,"15min":1.6020000000000001},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.147,"5min":13.069000000000001,"15min":13.069000000000001},"last":9.0500000000000007}]},{"osd":4,"last update":"Mon Oct 9 04:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.3819999999999999,"15min":1.4139999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":3.7629999999999999,"5min":6.4889999999999999,"15min":12.981},"last":3.7589999999999999},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.3520000000000001,"15min":1.4630000000000001},"min":{"1min":0.48199999999999998,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":8.8840000000000003,"5min":8.8840000000000003,"15min":9.4359999999999999},"last":0.93999999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":2.0190000000000001,"15min":1.8069999999999999},"min":{"1min":0.55100000000000005,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":8.8450000000000006,"5min":10.372,"15min":10.372},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.823,"15min":1.575},"min":{"1min":0.379,"5min":0.379,"15min":0.34200000000000003},"max":{"1min":8.9190000000000005,"5min":8.9190000000000005,"15min":9.4800000000000004},"last":0.96399999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.7010000000000001,"15min":1.4990000000000001},"min":{"1min":0.40600000000000003,"5min":0.36799999999999999,"15min":0.32100000000000001},"max":{"1min":5.6749999999999998,"5min":10.102,"15min":10.102},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.335,"15min":1.3620000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35399999999999998},"max":{"1min":5.4939999999999998,"5min":5.4939999999999998,"15min":8.8699999999999992},"last":3.694}]},{"osd":7,"last update":"Mon Oct 9 04:53:58 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.9590000000000001,"15min":1.7350000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33100000000000002},"max":{"1min":5.726,"5min":15.170999999999999,"15min":15.170999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.6240000000000001,"15min":1.627},"min":{"1min":0.49099999999999999,"5min":0.45100000000000001,"15min":0.36699999999999999},"max":{"1min":4.7489999999999997,"5min":9.2370000000000001,"15min":13.028},"last":3.7309999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019463,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":150,"seq":644245094645,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3062276,"kb_used_data":2106428,"kb_used_omap":6,"kb_used_meta":955833,"kb_avail":90682876,"statfs":{"total":95995035648,"available":92859265024,"internally_reserved":0,"allocated":2156982272,"data_stored":2156397835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":978773267},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":17.763999999999999,"5min":7.6349999999999998,"15min":3.7589999999999999},"min":{"1min":0.38400000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":363.64499999999998,"5min":363.64499999999998,"15min":363.64499999999998},"last":0.53100000000000003},{"interface":"front","average":{"1min":17.446999999999999,"5min":8.0510000000000002,"15min":4.0620000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":363.26900000000001,"5min":363.26900000000001,"15min":363.26900000000001},"last":1.012}]},{"osd":1,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":5.2009999999999996,"15min":2.903},"min":{"1min":0.308,"5min":0.308,"15min":0.28399999999999997},"max":{"1min":3.9820000000000002,"5min":130.762,"15min":130.762},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.044,"5min":5.6680000000000001,"15min":3.2480000000000002},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.26400000000000001},"max":{"1min":4.5279999999999996,"5min":130.86199999999999,"15min":130.86199999999999},"last":1.0569999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:54:08 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.0209999999999999,"15min":1.425},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.3740000000000001,"5min":7.1749999999999998,"15min":7.1920000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.401,"15min":1.889},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.625,"5min":7.1440000000000001,"15min":13.536},"last":0.81599999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":3.2080000000000002,"15min":2.3439999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.5720000000000001,"5min":130.69200000000001,"15min":130.69200000000001},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.286,"5min":3.2879999999999998,"15min":2.5179999999999998},"min":{"1min":0.443,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.5750000000000002,"5min":130.53399999999999,"15min":130.53399999999999},"last":0.97899999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.345,"15min":1.417},"min":{"1min":0.40200000000000002,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":1.702,"5min":12.734,"15min":12.734},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.115,"15min":1.262},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":1.484,"5min":8.3409999999999993,"15min":10.709},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.036,"15min":1.296},"min":{"1min":0.45200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.5820000000000001,"5min":7.2489999999999997,"15min":10.77},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.595,"15min":1.4990000000000001},"min":{"1min":0.41299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4409999999999998,"5min":14.223000000000001,"15min":14.223000000000001},"last":0.878}]},{"osd":7,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":2.1579999999999999,"15min":1.7909999999999999},"min":{"1min":0.46600000000000003,"5min":0.42499999999999999,"15min":0.32200000000000001},"max":{"1min":2.8250000000000002,"5min":12.706,"15min":12.706},"last":1.694},{"interface":"front","average":{"1min":1.012,"5min":1.7050000000000001,"15min":1.651},"min":{"1min":0.39600000000000002,"5min":0.39200000000000002,"15min":0.35699999999999998},"max":{"1min":2.3639999999999999,"5min":21.797000000000001,"15min":21.797000000000001},"last":1.095}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3502571520,"data_stored":3502571520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2146762752,"data_stored":2146762752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2109734912,"data_stored":2109734912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2868314112,"data_stored":2868314112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2164850688,"data_stored":2164850688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2494300160,"data_stored":2494300160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1825243136,"data_stored":1825243136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:55:01.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:55:01.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.371 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:01.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 75 16 93487 93471 77.8743 53.1875 0.020115 0.0128352 2023-10-09T04:55:01.475 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:01.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:01.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:01.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:01.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:01.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:02.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:02.029 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:02.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:02.153 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:02.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:02.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:02.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:02.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 76 16 94224 94208 77.4557 46.0625 0.144323 0.0128867 2023-10-09T04:55:02.415 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:02.417 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:02 smithi153 bash[59565]: cluster 2023-10-09T04:55:00.861856+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:55:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:02 smithi082 bash[77063]: cluster 2023-10-09T04:55:00.861856+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:55:02.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:02 smithi082 bash[79733]: cluster 2023-10-09T04:55:00.861856+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:55:02.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:02.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:02.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:02.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:02.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:02.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:02.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:02.939 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:02.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:03.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:03.068 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:03.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:03.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:03.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:03.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:03.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 77 16 95129 95113 77.1844 56.5625 0.0230663 0.0129502 2023-10-09T04:55:03.437 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:03.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:03.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:03.608 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:03.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:03.712 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:03.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:03.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:03.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:03.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:03.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:03.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:03.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:04.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:04.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:04.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:04.266 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:04.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:04.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:04.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:04.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 78 16 95976 95960 76.8735 52.9375 0.00814803 0.0130027 2023-10-09T04:55:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:04 smithi153 bash[59565]: cluster 2023-10-09T04:55:02.862557+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T04:55:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:04 smithi082 bash[77063]: cluster 2023-10-09T04:55:02.862557+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T04:55:04.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:04 smithi082 bash[79733]: cluster 2023-10-09T04:55:02.862557+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T04:55:04.505 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:04.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:04.634 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:04.667 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:04.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:04.809 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:04.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:04.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:04.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:04.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:05.041 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:05.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:05.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:05.174 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:05 smithi082 bash[77063]: audit 2023-10-09T04:55:05.127287+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.82:0/2778218518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:05.185 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:55:05.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:05.220 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:55:05.255 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:55:05.296 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:55:05.300 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:05.344 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:55:05.379 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:55:05.407 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 79 16 96806 96790 76.557 51.875 0.0143278 0.0130561 2023-10-09T04:55:05.413 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:55:05.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:05 smithi153 bash[59565]: audit 2023-10-09T04:55:05.127287+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.82:0/2778218518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:05.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.447 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:05 smithi082 bash[79733]: audit 2023-10-09T04:55:05.127287+0000 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.82:0/2778218518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:05.449 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.461 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:55:05.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:05.568 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.603 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:05.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:05.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:05.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:05.839 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:05.870 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:05.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:05.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:06.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:06.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:06.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:06.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:06.245 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:06 smithi082 bash[77063]: cluster 2023-10-09T04:55:04.863075+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T04:55:06.245 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:06 smithi082 bash[79733]: cluster 2023-10-09T04:55:04.863075+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T04:55:06.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:06.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:06.373 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:06 smithi153 bash[59565]: cluster 2023-10-09T04:55:04.863075+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T04:55:06.390 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:06.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:2023-10-09T04:55:06.407553+0000 min lat: 0.00123875 max lat: 0.508003 avg lat: 0.0131056 2023-10-09T04:55:06.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:55:06.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 80 16 97659 97643 76.2664 53.3125 0.00589676 0.0131056 2023-10-09T04:55:06.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:06.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:06.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:06.647 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:06.752 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:06.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:06.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:06.929 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:06.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:07.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:07.061 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:07.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:07.187 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:07.290 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:07.324 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:07.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 81 16 98323 98307 75.8372 41.5 0.0249374 0.013181 2023-10-09T04:55:07.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:07.453 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:07.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:07.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:07.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:07.715 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:07.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:07.846 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:07.953 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:07.980 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:07.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:08.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:08.195 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:55:08.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:08.360 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:08.370 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:08 smithi153 bash[59565]: cluster 2023-10-09T04:55:06.863639+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T04:55:08.370 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:08 smithi153 bash[59565]: audit 2023-10-09T04:55:07.222429+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.82:0/209653340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.370 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:08 smithi153 bash[59565]: audit 2023-10-09T04:55:07.222779+0000 mon.a (mon.0) 1802 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.386 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:08.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 82 16 99178 99162 75.564 53.4375 0.016731 0.0132277 2023-10-09T04:55:08.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[77063]: cluster 2023-10-09T04:55:06.863639+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T04:55:08.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[77063]: audit 2023-10-09T04:55:07.222429+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.82:0/209653340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[77063]: audit 2023-10-09T04:55:07.222779+0000 mon.a (mon.0) 1802 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[79733]: cluster 2023-10-09T04:55:06.863639+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T04:55:08.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[79733]: audit 2023-10-09T04:55:07.222429+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.82:0/209653340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:08 smithi082 bash[79733]: audit 2023-10-09T04:55:07.222779+0000 mon.a (mon.0) 1802 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:08.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:08.521 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:08.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.654 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:08.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:08.799 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.802 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.35442.0:192852 8.9 8:9cddcae4:::benchmark_data_smithi082_175677_object75838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 542.27500411300002, 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31415913600000001, 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.35442", 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/464232351", 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 192852 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.803 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522383+0000", 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.804 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522382+0000", 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7500000000000003e-06 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522390+0000", 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.377e-05 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522403+0000", 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.9600000000000006e-06 2023-10-09T04:55:08.805 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522412+0000", 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8878999999999998e-05 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522451+0000", 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7504999999999996e-05 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.806 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522549+0000", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.7978000000000003e-05 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.522647+0000", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040000080000000002 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.526647+0000", 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00037852200000000001 2023-10-09T04:55:08.807 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.527025+0000", 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.30938406600000001 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836409+0000", 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2324999999999995e-05 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.808 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836492+0000", 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0847999999999998e-05 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:46:06.836543+0000", 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.31415913600000001 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.809 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21630 10.a 10:524a61f3:::benchmark_data_smithi082_223129_object21629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.406783+0000", 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.390604785999997, 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42698707499999999, 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21630 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.810 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406783+0000", 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406783+0000", 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.811 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406782+0000", 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2249000000000001e-05 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406834+0000", 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2380000000000003e-06 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406843+0000", 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1969999999999999e-06 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.812 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.406851+0000", 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00066808099999999995 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407519+0000", 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00022728000000000001 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.813 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407746+0000", 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00027360299999999998 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.408019+0000", 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0039260989999999997 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411946+0000", 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00072159000000000004 2023-10-09T04:55:08.814 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.412667+0000", 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42100099099999999 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833668+0000", 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8349000000000001e-05 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.815 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833716+0000", 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3306000000000001e-05 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833770+0000", 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42698707499999999 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.816 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21633 10.d 10:baff69f7:::benchmark_data_smithi082_223129_object21632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.407589+0000", 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.389798182000007, 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42618047199999998, 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21633 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.817 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407589+0000", 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407589+0000", 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.818 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407589+0000", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.4804e-05 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407633+0000", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1182000000000001e-05 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407645+0000", 2023-10-09T04:55:08.819 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.613e-06 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407651+0000", 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5707000000000001e-05 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407697+0000", 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000114189 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.820 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407811+0000", 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000145702 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.407957+0000", 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002109044 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.821 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410066+0000", 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001872182 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411938+0000", 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42168698199999999 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833625+0000", 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0483999999999997e-05 2023-10-09T04:55:08.822 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833706+0000", 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4269000000000004e-05 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.833770+0000", 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42618047199999998 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.823 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21637 10.d 10:b5b8c44e:::benchmark_data_smithi082_223129_object21636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.410053+0000", 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.387334138, 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42930855899999998, 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21637 2023-10-09T04:55:08.824 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410053+0000", 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410053+0000", 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.825 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410052+0000", 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8305e-05 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410090+0000", 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3049e-05 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.826 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410104+0000", 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9899999999999997e-06 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410112+0000", 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8967999999999997e-05 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410151+0000", 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014077100000000001 2023-10-09T04:55:08.827 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410291+0000", 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017899100000000001 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410470+0000", 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0014780939999999999 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.828 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411948+0000", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000385852 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.412334+0000", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42691725899999999 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839251+0000", 2023-10-09T04:55:08.829 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0538000000000002e-05 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839312+0000", 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9925000000000001e-05 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839362+0000", 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42930855899999998 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.830 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21641 10.b 10:dd73d6b2:::benchmark_data_smithi082_223129_object21640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.410698+0000", 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.386689813000004, 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.426448942, 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.831 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21641 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410698+0000", 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410698+0000", 2023-10-09T04:55:08.832 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410697+0000", 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012961799999999999 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410826+0000", 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9675999999999999e-05 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.833 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410846+0000", 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8559999999999996e-06 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.410855+0000", 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00038283099999999998 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.834 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411238+0000", 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013169200000000001 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411369+0000", 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015234800000000001 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.411522+0000", 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001135275 2023-10-09T04:55:08.835 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.412657+0000", 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0020331519999999999 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.414690+0000", 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42227476899999999 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.836 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.836965+0000", 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6839e-05 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.837052+0000", 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4827999999999994e-05 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.837147+0000", 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.426448942 2023-10-09T04:55:08.837 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21647 10.b 10:d20f5c64:::benchmark_data_smithi082_223129_object21646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.413498+0000", 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.383889116000006, 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42574741300000002, 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.838 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21647 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413498+0000", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413498+0000", 2023-10-09T04:55:08.839 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413497+0000", 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.7759e-05 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413555+0000", 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4073999999999999e-05 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.840 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413569+0000", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.759e-06 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413576+0000", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3829000000000002e-05 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413630+0000", 2023-10-09T04:55:08.841 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015073800000000001 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413780+0000", 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000139729 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.413920+0000", 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00093700700000000001 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.842 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.414857+0000", 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0010898380000000001 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.415947+0000", 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.423175048 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.843 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839122+0000", 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4919999999999995e-05 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839187+0000", 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.8918000000000001e-05 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839246+0000", 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42574741300000002 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.844 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21651 10.d 10:b218a2f8:::benchmark_data_smithi082_223129_object21650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.416290+0000", 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.381097639999993, 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42321384099999998, 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.845 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21651 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416290+0000", 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416290+0000", 2023-10-09T04:55:08.846 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416288+0000", 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.0226000000000002e-05 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416359+0000", 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9236e-05 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.847 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416378+0000", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3119999999999999e-06 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416384+0000", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6125000000000003e-05 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416430+0000", 2023-10-09T04:55:08.848 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016900199999999999 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416599+0000", 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00017463099999999999 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416774+0000", 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001111288 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.849 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.417885+0000", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00062792999999999996 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.418513+0000", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42089395800000001 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839407+0000", 2023-10-09T04:55:08.850 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.9626999999999998e-05 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839457+0000", 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6956000000000002e-05 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839504+0000", 2023-10-09T04:55:08.851 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42321384099999998 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:21652 10.b 10:d39f2c64:::benchmark_data_smithi082_223129_object21651:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:53:56.416562+0000", 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 72.380825677999994, 2023-10-09T04:55:08.853 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42309913100000002, 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 21652 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416562+0000", 2023-10-09T04:55:08.854 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416562+0000", 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416561+0000", 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.0272999999999997e-05 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.855 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416621+0000", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0501e-05 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416632+0000", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6450000000000004e-06 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416636+0000", 2023-10-09T04:55:08.856 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8770999999999999e-05 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416675+0000", 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000125234 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.416800+0000", 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021735400000000001 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.857 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.417018+0000", 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00099086599999999997 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.418009+0000", 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000965776 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.858 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.418974+0000", 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42058646799999999 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839561+0000", 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3776000000000001e-05 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839615+0000", 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6236999999999998e-05 2023-10-09T04:55:08.859 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:53:56.839661+0000", 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.42309913100000002 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59963 10.f 10:f3d60733:::benchmark_data_smithi082_223129_object59962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.288703+0000", 2023-10-09T04:55:08.860 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.508684588000001, 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.311972162, 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59963 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.861 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288703+0000", 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288703+0000", 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288702+0000", 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.5701999999999997e-05 2023-10-09T04:55:08.862 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288748+0000", 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.9999999999999996e-06 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288756+0000", 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3970000000000002e-06 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.863 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288761+0000", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3761999999999999e-05 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288805+0000", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000152158 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.288957+0000", 2023-10-09T04:55:08.864 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000191876 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.289149+0000", 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027291260000000001 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.291878+0000", 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0033777490000000002 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.865 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.295256+0000", 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.30525151700000003 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600507+0000", 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.2966999999999995e-05 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.866 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600570+0000", 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000104844 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600675+0000", 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.311972162 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.867 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59972 10.e 10:71723b14:::benchmark_data_smithi082_223129_object59971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.300345+0000", 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.497042270000001, 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28852118100000002, 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59972 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.868 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300345+0000", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300345+0000", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300344+0000", 2023-10-09T04:55:08.869 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5728000000000001e-05 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300380+0000", 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0608000000000001e-05 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300390+0000", 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6540000000000002e-06 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.870 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300397+0000", 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3965000000000003e-05 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300451+0000", 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016453 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.871 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300615+0000", 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015203999999999999 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.300767+0000", 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00203652 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302804+0000", 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0034395419999999999 2023-10-09T04:55:08.872 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.306243+0000", 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28230523200000002 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.588549+0000", 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00018626499999999999 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.588735+0000", 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000131538 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.588866+0000", 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28852118100000002 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.874 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59976 10.f 10:fc5f13f7:::benchmark_data_smithi082_223129_object59975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.302790+0000", 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.494597865999999, 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29803342999999999, 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59976 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.875 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302790+0000", 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302790+0000", 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.876 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302789+0000", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4408e-05 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302823+0000", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3820000000000003e-06 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302829+0000", 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.3699999999999999e-06 2023-10-09T04:55:08.877 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302832+0000", 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6514999999999998e-05 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.302879+0000", 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014773600000000001 2023-10-09T04:55:08.878 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303026+0000", 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0031397780000000002 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.306166+0000", 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0044906039999999996 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.879 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.310657+0000", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0030144199999999999 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.313671+0000", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.287053059 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600724+0000", 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5500000000000001e-05 2023-10-09T04:55:08.880 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600780+0000", 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3470000000000002e-05 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600823+0000", 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29803342999999999 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.881 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59979 10.e 10:7de0e38b:::benchmark_data_smithi082_223129_object59978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.303392+0000", 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.493995812000001, 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.285865639, 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.882 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59979 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303392+0000", 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303392+0000", 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.883 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303391+0000", 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011861 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303509+0000", 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1039000000000001e-05 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.884 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303520+0000", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9820000000000002e-06 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303527+0000", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9125000000000002e-05 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303566+0000", 2023-10-09T04:55:08.885 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014352700000000001 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303710+0000", 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000110611 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303820+0000", 2023-10-09T04:55:08.886 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027055270000000001 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.306526+0000", 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0016466110000000001 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.308173+0000", 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28081148099999997 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.887 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.588984+0000", 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014444300000000001 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.589129+0000", 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012880300000000001 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.888 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.589257+0000", 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.285865639 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59983 10.f 10:f6486672:::benchmark_data_smithi082_223129_object59982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.303545+0000", 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.493842225000002, 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.297467547, 2023-10-09T04:55:08.889 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59983 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303545+0000", 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.890 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303545+0000", 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303544+0000", 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5386999999999998e-05 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.891 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303580+0000", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.909e-06 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303588+0000", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3759999999999999e-06 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.303593+0000", 2023-10-09T04:55:08.892 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0049364049999999996 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.308529+0000", 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000101406 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.308631+0000", 2023-10-09T04:55:08.893 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000118913 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.308750+0000", 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002171348 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.310921+0000", 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.004681396 2023-10-09T04:55:08.894 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.315602+0000", 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28533834600000002 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600941+0000", 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6068e-05 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.895 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.600987+0000", 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5928999999999999e-05 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601013+0000", 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.297467547 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.896 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59987 10.f 10:f47c9871:::benchmark_data_smithi082_223129_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.311146+0000", 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.486241419999999, 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29004974500000003, 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.898 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59987 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311146+0000", 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311146+0000", 2023-10-09T04:55:08.899 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311145+0000", 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.1199999999999997e-05 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311206+0000", 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.243e-05 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.900 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311219+0000", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6089999999999997e-06 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311225+0000", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.0549000000000001e-05 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311266+0000", 2023-10-09T04:55:08.901 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000121729 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311387+0000", 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000147632 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.902 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.311535+0000", 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0040831260000000003 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.315618+0000", 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0014620200000000001 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.317080+0000", 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28399976599999999 2023-10-09T04:55:08.903 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601080+0000", 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5147000000000005e-05 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601145+0000", 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0738000000000001e-05 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.904 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601196+0000", 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29004974500000003 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:59992 10.f 10:fe85891f:::benchmark_data_smithi082_223129_object59991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:25.321816+0000", 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 43.475571733000002, 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27948146400000001, 2023-10-09T04:55:08.905 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 59992 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321816+0000", 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.906 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321816+0000", 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321814+0000", 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7123000000000001e-05 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.907 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321881+0000", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8091999999999999e-05 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321900+0000", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.619e-06 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321908+0000", 2023-10-09T04:55:08.908 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.8729000000000001e-05 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.321947+0000", 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011488 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.322062+0000", 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00013867400000000001 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.909 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.322200+0000", 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002168199 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.324369+0000", 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00059176900000000002 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.910 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.324960+0000", 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27628031800000002 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601241+0000", 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2753999999999998e-05 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601274+0000", 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3711999999999999e-05 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.911 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:25.601297+0000", 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27948146400000001 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:67318 10.0 10:01cd8a37:::benchmark_data_smithi082_223129_object67317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:32.460199+0000", 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 36.337188918000003, 2023-10-09T04:55:08.912 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28186513499999999, 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 67318 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460199+0000", 2023-10-09T04:55:08.913 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460199+0000", 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460197+0000", 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3196999999999999e-05 2023-10-09T04:55:08.914 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460250+0000", 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2286000000000001e-05 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460262+0000", 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2169999999999997e-06 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.915 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460270+0000", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7712e-05 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460317+0000", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014697700000000001 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460464+0000", 2023-10-09T04:55:08.916 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015214800000000001 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.460617+0000", 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026955709999999999 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463312+0000", 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000214885 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.917 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463527+0000", 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27840267800000001 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.741930+0000", 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.4856000000000004e-05 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.918 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.741995+0000", 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.9216999999999998e-05 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742064+0000", 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.28186513499999999 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:67324 10.e 10:7c694db9:::benchmark_data_smithi082_223129_object67323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.919 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:32.463331+0000", 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 36.334056844999999, 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27878141099999998, 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 67324 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.920 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463331+0000", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463331+0000", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463329+0000", 2023-10-09T04:55:08.921 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2112e-05 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463361+0000", 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0543000000000001e-05 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463372+0000", 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.1869999999999996e-06 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.922 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463377+0000", 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3195999999999998e-05 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463420+0000", 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5846000000000004e-05 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.923 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463506+0000", 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.3394999999999996e-05 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.463599+0000", 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0026659660000000001 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466265+0000", 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00206361 2023-10-09T04:55:08.924 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468329+0000", 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27367418199999999 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742003+0000", 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.6931000000000001e-05 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.925 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742060+0000", 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.2095000000000002e-05 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742112+0000", 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27878141099999998 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.926 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:67326 10.7 10:e9cad869:::benchmark_data_smithi082_223129_object67325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:32.466060+0000", 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 36.331327533, 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.279080999, 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 67326 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.927 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466060+0000", 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466060+0000", 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.928 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466058+0000", 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.388e-05 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466102+0000", 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3648e-05 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466116+0000", 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.7170000000000002e-06 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466123+0000", 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4625999999999997e-05 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466157+0000", 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9746000000000003e-05 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.930 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466257+0000", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000128498 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466386+0000", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27601899099999999 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742405+0000", 2023-10-09T04:55:08.931 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.1495000000000002e-05 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742446+0000", 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0025986469999999999 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.745045+0000", 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0213000000000002e-05 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.932 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.745095+0000", 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.6084000000000001e-05 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.745141+0000", 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.279080999 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.933 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:67327 10.d 10:b2e3dd16:::benchmark_data_smithi082_223129_object67326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:32.466208+0000", 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 36.331179370999998, 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27611044600000001, 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 67327 2023-10-09T04:55:08.934 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466208+0000", 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466208+0000", 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.935 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466207+0000", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8788999999999995e-05 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466276+0000", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4300000000000006e-06 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466284+0000", 2023-10-09T04:55:08.936 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2239999999999997e-06 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466288+0000", 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.3509e-05 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466332+0000", 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000120745 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.937 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466453+0000", 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00014156499999999999 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.466594+0000", 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27520716899999997 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.938 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.741801+0000", 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000394762 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742196+0000", 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.4261999999999998e-05 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742230+0000", 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8248000000000003e-05 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.939 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742279+0000", 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9926999999999999e-05 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.742319+0000", 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27611044600000001 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.940 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:67328 10.f 10:f66a5972:::benchmark_data_smithi082_223129_object67327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:54:32.468474+0000", 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 36.328913948999997, 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29112238200000001, 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 67328 2023-10-09T04:55:08.941 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468474+0000", 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468474+0000", 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.942 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468473+0000", 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6507999999999994e-05 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468539+0000", 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6660000000000008e-06 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:08.943 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468547+0000", 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.7190000000000001e-06 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468551+0000", 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.3655999999999998e-05 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468585+0000", 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.5228999999999993e-05 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.944 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468670+0000", 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5519999999999993e-05 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.468766+0000", 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.27627041800000002 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.945 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.745036+0000", 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000146413 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.745183+0000", 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.01432308 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "op_commit", 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.759506+0000", 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.0478e-05 2023-10-09T04:55:08.946 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "commit_sent", 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.759556+0000", 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.9743999999999998e-05 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:54:32.759596+0000", 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.29112238200000001 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.947 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:08.948 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:08.948 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:55:08.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:09.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:09.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.084 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:09.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:09.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:09.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:09.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:55:09.314 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:09.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.175932+0000 mon.a (mon.0) 1803 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:09.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.175975+0000 mon.a (mon.0) 1804 : cluster 1 Cluster is now healthy 2023-10-09T04:55:09.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.178251+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.181429+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: audit 2023-10-09T04:55:08.186679+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.186963+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.187875+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.188001+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.196671+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.318 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.175932+0000 mon.a (mon.0) 1803 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.175975+0000 mon.a (mon.0) 1804 : cluster 1 Cluster is now healthy 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.178251+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.181429+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: audit 2023-10-09T04:55:08.186679+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.186963+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.187875+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.188001+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.319 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.196671+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.357 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:09.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 83 16 99994 99978 75.268 51 0.00739598 0.0132797 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.175932+0000 mon.a (mon.0) 1803 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.175975+0000 mon.a (mon.0) 1804 : cluster 1 Cluster is now healthy 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.178251+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.181429+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: audit 2023-10-09T04:55:08.186679+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.186963+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.187875+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.188001+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:09 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.196671+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T04:55:09.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:09.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:09.587 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:09.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:09.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:09.745 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:09.853 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:09.876 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:09.881 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:09.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:09.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:09.912 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:09] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-10-09T04:55:09.985 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:10.061 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:55:10.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:10.266 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.293 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:10.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:10.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 84 16 100762 100746 74.9434 48 0.0115221 0.013338 2023-10-09T04:55:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:10 smithi153 bash[59565]: cluster 2023-10-09T04:55:08.864117+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:55:10.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:10.433 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:10 smithi082 bash[77063]: cluster 2023-10-09T04:55:08.864117+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:55:10.433 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:10 smithi082 bash[79733]: cluster 2023-10-09T04:55:08.864117+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:55:10.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:10.575 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:10.682 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:10.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:10.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.840 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:10.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.976 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:10.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:11.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:11.109 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:11.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:11.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:11.245 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:11.351 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:11.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:11.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 85 16 101574 101558 74.6587 50.75 0.0308053 0.0133877 2023-10-09T04:55:11.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:11.505 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:11.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.634 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:11.655 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:55:11.740 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.769 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:11.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.899 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:11.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:12.003 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:12.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:12.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.160 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:12.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:12.294 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:12.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:12.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:12 smithi082 bash[77063]: cluster 2023-10-09T04:55:10.864689+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s 2023-10-09T04:55:12.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:12 smithi082 bash[79733]: cluster 2023-10-09T04:55:10.864689+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s 2023-10-09T04:55:12.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 86 16 102384 102368 74.3793 50.625 0.0119823 0.0134383 2023-10-09T04:55:12.422 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:12.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:12.541 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:12 smithi153 bash[59565]: cluster 2023-10-09T04:55:10.864689+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s 2023-10-09T04:55:12.556 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:12.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.699 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:12.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:12.838 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:12.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:12.979 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:12.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:13.083 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:13.238 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:13.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:13.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:13.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:13.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:13.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 87 16 103165 103149 74.0854 48.8125 0.0207108 0.0134922 2023-10-09T04:55:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:13.510 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:13.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:13.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:13.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:13.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:13.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:13.957 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:13.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:14.061 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:14.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:14.201 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:14.227 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:14.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:14.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:14.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:14.408 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 88 16 103916 103900 73.7769 46.9375 0.028084 0.0135486 2023-10-09T04:55:14.457 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:14.464 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:14 smithi153 bash[59565]: cluster 2023-10-09T04:55:12.865283+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T04:55:14.479 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:14.481 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:55:14.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:14 smithi082 bash[77063]: cluster 2023-10-09T04:55:12.865283+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T04:55:14.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:14 smithi082 bash[79733]: cluster 2023-10-09T04:55:12.865283+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-09T04:55:14.486 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:55:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:14.587 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:14.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:14.724 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:14.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:14.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:14.860 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:14.879 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:14.892 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:14.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:14.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:15.022 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:15.129 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:15.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:15.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:15.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:15.401 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[77063]: audit 2023-10-09T04:55:14.303756+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[77063]: audit 2023-10-09T04:55:14.305876+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[77063]: audit 2023-10-09T04:55:14.461331+0000 mgr.y (mgr.34107) 1238 : audit 0 from='client.45860 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[77063]: cluster 2023-10-09T04:55:14.868609+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[79733]: audit 2023-10-09T04:55:14.303756+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[79733]: audit 2023-10-09T04:55:14.305876+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[79733]: audit 2023-10-09T04:55:14.461331+0000 mgr.y (mgr.34107) 1238 : audit 0 from='client.45860 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:15.408 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:15 smithi082 bash[79733]: cluster 2023-10-09T04:55:14.868609+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T04:55:15.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout: 89 15 104706 104691 73.5034 49.4375 0.0369877 0.0135994 2023-10-09T04:55:15.445 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:15.549 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:15.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:15 smithi153 bash[59565]: audit 2023-10-09T04:55:14.303756+0000 mon.a (mon.0) 1807 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:15 smithi153 bash[59565]: audit 2023-10-09T04:55:14.305876+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:15 smithi153 bash[59565]: audit 2023-10-09T04:55:14.461331+0000 mgr.y (mgr.34107) 1238 : audit 0 from='client.45860 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:15 smithi153 bash[59565]: cluster 2023-10-09T04:55:14.868609+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1359: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T04:55:15.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:15.724 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:15.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:15.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:15.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:15.865 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1358,"stamp":"2023-10-09T04:55:12.864876+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6604199859,"num_objects":100727,"num_object_clones":0,"num_object_copies":302181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100727,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":101805,"num_write_kb":6459895,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":8398438,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101869,"ondisk_log_size":101869,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":24652876,"kb_used_data":19301972,"kb_used_omap":47,"kb_used_meta":5350800,"kb_avail":631563188,"statfs":{"total":671965249536,"available":646720704512,"internally_reserved":0,"allocated":19765219328,"data_stored":19758766568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":5479219613},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,6,13,12,4,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":529989632,"num_objects":8087,"num_object_clones":0,"num_object_copies":24261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8087,"num_write_kb":517568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002797"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":40,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.246959+0000","last_change":"2023-10-09T04:51:14.108186+0000","last_active":"2023-10-09T04:54:47.246959+0000","last_peered":"2023-10-09T04:54:47.246959+0000","last_clean":"2023-10-09T04:54:47.246959+0000","last_became_active":"2023-10-09T04:51:14.107642+0000","last_became_peered":"2023-10-09T04:51:14.107642+0000","last_unstale":"2023-10-09T04:54:47.246959+0000","last_undegraded":"2023-10-09T04:54:47.246959+0000","last_fullsized":"2023-10-09T04:54:47.246959+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:32:39.703389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":274,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.597471+0000","last_change":"2023-10-09T04:51:14.111673+0000","last_active":"2023-10-09T04:54:52.597471+0000","last_peered":"2023-10-09T04:54:52.597471+0000","last_clean":"2023-10-09T04:54:52.597471+0000","last_became_active":"2023-10-09T04:51:14.111303+0000","last_became_peered":"2023-10-09T04:51:14.111303+0000","last_unstale":"2023-10-09T04:54:52.597471+0000","last_undegraded":"2023-10-09T04:54:52.597471+0000","last_fullsized":"2023-10-09T04:54:52.597471+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:38.607841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":271,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229445+0000","last_change":"2023-10-09T04:51:14.150395+0000","last_active":"2023-10-09T04:54:47.229445+0000","last_peered":"2023-10-09T04:54:47.229445+0000","last_clean":"2023-10-09T04:54:47.229445+0000","last_became_active":"2023-10-09T04:51:14.149329+0000","last_became_peered":"2023-10-09T04:51:14.149329+0000","last_unstale":"2023-10-09T04:54:47.229445+0000","last_undegraded":"2023-10-09T04:54:47.229445+0000","last_fullsized":"2023-10-09T04:54:47.229445+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:50.627469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":287,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248728+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:54:47.248728+0000","last_peered":"2023-10-09T04:54:47.248728+0000","last_clean":"2023-10-09T04:54:47.248728+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:54:47.248728+0000","last_undegraded":"2023-10-09T04:54:47.248728+0000","last_fullsized":"2023-10-09T04:54:47.248728+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":289,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.244894+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:54:47.244894+0000","last_peered":"2023-10-09T04:54:47.244894+0000","last_clean":"2023-10-09T04:54:47.244894+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:54:47.244894+0000","last_undegraded":"2023-10-09T04:54:47.244894+0000","last_fullsized":"2023-10-09T04:54:47.244894+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.241054+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:54:47.241054+0000","last_peered":"2023-10-09T04:54:47.241054+0000","last_clean":"2023-10-09T04:54:47.241054+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:54:47.241054+0000","last_undegraded":"2023-10-09T04:54:47.241054+0000","last_fullsized":"2023-10-09T04:54:47.241054+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":281,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247952+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:54:47.247952+0000","last_peered":"2023-10-09T04:54:47.247952+0000","last_clean":"2023-10-09T04:54:47.247952+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:54:47.247952+0000","last_undegraded":"2023-10-09T04:54:47.247952+0000","last_fullsized":"2023-10-09T04:54:47.247952+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229588+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:54:47.229588+0000","last_peered":"2023-10-09T04:54:47.229588+0000","last_clean":"2023-10-09T04:54:47.229588+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:54:47.229588+0000","last_undegraded":"2023-10-09T04:54:47.229588+0000","last_fullsized":"2023-10-09T04:54:47.229588+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232377+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:54:47.232377+0000","last_peered":"2023-10-09T04:54:47.232377+0000","last_clean":"2023-10-09T04:54:47.232377+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:54:47.232377+0000","last_undegraded":"2023-10-09T04:54:47.232377+0000","last_fullsized":"2023-10-09T04:54:47.232377+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239302+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:54:47.239302+0000","last_peered":"2023-10-09T04:54:47.239302+0000","last_clean":"2023-10-09T04:54:47.239302+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:54:47.239302+0000","last_undegraded":"2023-10-09T04:54:47.239302+0000","last_fullsized":"2023-10-09T04:54:47.239302+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":143,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239418+0000","last_change":"2023-10-09T04:51:14.339838+0000","last_active":"2023-10-09T04:54:47.239418+0000","last_peered":"2023-10-09T04:54:47.239418+0000","last_clean":"2023-10-09T04:54:47.239418+0000","last_became_active":"2023-10-09T04:51:14.150185+0000","last_became_peered":"2023-10-09T04:51:14.150185+0000","last_unstale":"2023-10-09T04:54:47.239418+0000","last_undegraded":"2023-10-09T04:54:47.239418+0000","last_fullsized":"2023-10-09T04:54:47.239418+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":250,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240291+0000","last_change":"2023-10-09T04:50:44.052710+0000","last_active":"2023-10-09T04:54:47.240291+0000","last_peered":"2023-10-09T04:54:47.240291+0000","last_clean":"2023-10-09T04:54:47.240291+0000","last_became_active":"2023-10-09T04:50:44.052312+0000","last_became_peered":"2023-10-09T04:50:44.052312+0000","last_unstale":"2023-10-09T04:54:47.240291+0000","last_undegraded":"2023-10-09T04:54:47.240291+0000","last_fullsized":"2023-10-09T04:54:47.240291+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":250,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239731+0000","last_change":"2023-10-09T04:51:14.162470+0000","last_active":"2023-10-09T04:54:47.239731+0000","last_peered":"2023-10-09T04:54:47.239731+0000","last_clean":"2023-10-09T04:54:47.239731+0000","last_became_active":"2023-10-09T04:51:14.129144+0000","last_became_peered":"2023-10-09T04:51:14.129144+0000","last_unstale":"2023-10-09T04:54:47.239731+0000","last_undegraded":"2023-10-09T04:54:47.239731+0000","last_fullsized":"2023-10-09T04:54:47.239731+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240511+0000","last_change":"2023-10-09T04:51:14.113752+0000","last_active":"2023-10-09T04:54:47.240511+0000","last_peered":"2023-10-09T04:54:47.240511+0000","last_clean":"2023-10-09T04:54:47.240511+0000","last_became_active":"2023-10-09T04:51:14.113290+0000","last_became_peered":"2023-10-09T04:51:14.113290+0000","last_unstale":"2023-10-09T04:54:47.240511+0000","last_undegraded":"2023-10-09T04:54:47.240511+0000","last_fullsized":"2023-10-09T04:54:47.240511+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":158,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239418+0000","last_change":"2023-10-09T04:51:14.179666+0000","last_active":"2023-10-09T04:54:47.239418+0000","last_peered":"2023-10-09T04:54:47.239418+0000","last_clean":"2023-10-09T04:54:47.239418+0000","last_became_active":"2023-10-09T04:51:14.130965+0000","last_became_peered":"2023-10-09T04:51:14.130965+0000","last_unstale":"2023-10-09T04:54:47.239418+0000","last_undegraded":"2023-10-09T04:54:47.239418+0000","last_fullsized":"2023-10-09T04:54:47.239418+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":209,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239740+0000","last_change":"2023-10-09T04:50:44.064722+0000","last_active":"2023-10-09T04:54:47.239740+0000","last_peered":"2023-10-09T04:54:47.239740+0000","last_clean":"2023-10-09T04:54:47.239740+0000","last_became_active":"2023-10-09T04:50:44.064373+0000","last_became_peered":"2023-10-09T04:50:44.064373+0000","last_unstale":"2023-10-09T04:54:47.239740+0000","last_undegraded":"2023-10-09T04:54:47.239740+0000","last_fullsized":"2023-10-09T04:54:47.239740+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":285,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:55:07.079892+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:55:07.079892+0000","last_peered":"2023-10-09T04:55:07.079892+0000","last_clean":"2023-10-09T04:55:07.079892+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:55:07.079892+0000","last_undegraded":"2023-10-09T04:55:07.079892+0000","last_fullsized":"2023-10-09T04:55:07.079892+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232357+0000","last_change":"2023-10-09T04:53:31.658083+0000","last_active":"2023-10-09T04:54:47.232357+0000","last_peered":"2023-10-09T04:54:47.232357+0000","last_clean":"2023-10-09T04:54:47.232357+0000","last_became_active":"2023-10-09T04:53:31.657030+0000","last_became_peered":"2023-10-09T04:53:31.657030+0000","last_unstale":"2023-10-09T04:54:47.232357+0000","last_undegraded":"2023-10-09T04:54:47.232357+0000","last_fullsized":"2023-10-09T04:54:47.232357+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":131,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239411+0000","last_change":"2023-10-09T04:51:14.129515+0000","last_active":"2023-10-09T04:54:47.239411+0000","last_peered":"2023-10-09T04:54:47.239411+0000","last_clean":"2023-10-09T04:54:47.239411+0000","last_became_active":"2023-10-09T04:51:14.128945+0000","last_became_peered":"2023-10-09T04:51:14.128945+0000","last_unstale":"2023-10-09T04:54:47.239411+0000","last_undegraded":"2023-10-09T04:54:47.239411+0000","last_fullsized":"2023-10-09T04:54:47.239411+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":284,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248811+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:54:47.248811+0000","last_peered":"2023-10-09T04:54:47.248811+0000","last_clean":"2023-10-09T04:54:47.248811+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:54:47.248811+0000","last_undegraded":"2023-10-09T04:54:47.248811+0000","last_fullsized":"2023-10-09T04:54:47.248811+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":271,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240212+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:54:47.240212+0000","last_peered":"2023-10-09T04:54:47.240212+0000","last_clean":"2023-10-09T04:54:47.240212+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:54:47.240212+0000","last_undegraded":"2023-10-09T04:54:47.240212+0000","last_fullsized":"2023-10-09T04:54:47.240212+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231873+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:54:47.231873+0000","last_peered":"2023-10-09T04:54:47.231873+0000","last_clean":"2023-10-09T04:54:47.231873+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:54:47.231873+0000","last_undegraded":"2023-10-09T04:54:47.231873+0000","last_fullsized":"2023-10-09T04:54:47.231873+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"307'6277","reported_seq":6299,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:09.963413+0000","last_change":"2023-10-09T04:53:24.353425+0000","last_active":"2023-10-09T04:55:09.963413+0000","last_peered":"2023-10-09T04:55:09.963413+0000","last_clean":"2023-10-09T04:55:09.963413+0000","last_became_active":"2023-10-09T04:53:24.352877+0000","last_became_peered":"2023-10-09T04:53:24.352877+0000","last_unstale":"2023-10-09T04:55:09.963413+0000","last_undegraded":"2023-10-09T04:55:09.963413+0000","last_fullsized":"2023-10-09T04:55:09.963413+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6277,"log_dups_size":0,"ondisk_log_size":6277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411369472,"num_objects":6277,"num_object_clones":0,"num_object_copies":18831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6277,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":139,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240007+0000","last_change":"2023-10-09T04:51:14.384573+0000","last_active":"2023-10-09T04:54:47.240007+0000","last_peered":"2023-10-09T04:54:47.240007+0000","last_clean":"2023-10-09T04:54:47.240007+0000","last_became_active":"2023-10-09T04:51:14.155381+0000","last_became_peered":"2023-10-09T04:51:14.155381+0000","last_unstale":"2023-10-09T04:54:47.240007+0000","last_undegraded":"2023-10-09T04:54:47.240007+0000","last_fullsized":"2023-10-09T04:54:47.240007+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231828+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:54:47.231828+0000","last_peered":"2023-10-09T04:54:47.231828+0000","last_clean":"2023-10-09T04:54:47.231828+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:54:47.231828+0000","last_undegraded":"2023-10-09T04:54:47.231828+0000","last_fullsized":"2023-10-09T04:54:47.231828+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"307'6262","reported_seq":6284,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.638565+0000","last_change":"2023-10-09T04:53:24.139931+0000","last_active":"2023-10-09T04:55:08.638565+0000","last_peered":"2023-10-09T04:55:08.638565+0000","last_clean":"2023-10-09T04:55:08.638565+0000","last_became_active":"2023-10-09T04:53:24.139560+0000","last_became_peered":"2023-10-09T04:53:24.139560+0000","last_unstale":"2023-10-09T04:55:08.638565+0000","last_undegraded":"2023-10-09T04:55:08.638565+0000","last_fullsized":"2023-10-09T04:55:08.638565+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6262,"log_dups_size":0,"ondisk_log_size":6262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410386432,"num_objects":6262,"num_object_clones":0,"num_object_copies":18786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6262,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":152,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240037+0000","last_change":"2023-10-09T04:51:14.431351+0000","last_active":"2023-10-09T04:54:47.240037+0000","last_peered":"2023-10-09T04:54:47.240037+0000","last_clean":"2023-10-09T04:54:47.240037+0000","last_became_active":"2023-10-09T04:51:14.156447+0000","last_became_peered":"2023-10-09T04:51:14.156447+0000","last_unstale":"2023-10-09T04:54:47.240037+0000","last_undegraded":"2023-10-09T04:54:47.240037+0000","last_fullsized":"2023-10-09T04:54:47.240037+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":113,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229309+0000","last_change":"2023-10-09T04:51:14.150212+0000","last_active":"2023-10-09T04:54:47.229309+0000","last_peered":"2023-10-09T04:54:47.229309+0000","last_clean":"2023-10-09T04:54:47.229309+0000","last_became_active":"2023-10-09T04:51:14.149425+0000","last_became_peered":"2023-10-09T04:51:14.149425+0000","last_unstale":"2023-10-09T04:54:47.229309+0000","last_undegraded":"2023-10-09T04:54:47.229309+0000","last_fullsized":"2023-10-09T04:54:47.229309+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.783749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"307'6387","reported_seq":6410,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.062254+0000","last_change":"2023-10-09T04:53:24.349405+0000","last_active":"2023-10-09T04:55:12.062254+0000","last_peered":"2023-10-09T04:55:12.062254+0000","last_clean":"2023-10-09T04:55:12.062254+0000","last_became_active":"2023-10-09T04:53:24.348702+0000","last_became_peered":"2023-10-09T04:53:24.348702+0000","last_unstale":"2023-10-09T04:55:12.062254+0000","last_undegraded":"2023-10-09T04:55:12.062254+0000","last_fullsized":"2023-10-09T04:55:12.062254+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:31.168505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418578432,"num_objects":6387,"num_object_clones":0,"num_object_copies":19161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":126,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239521+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:54:47.239521+0000","last_peered":"2023-10-09T04:54:47.239521+0000","last_clean":"2023-10-09T04:54:47.239521+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:54:47.239521+0000","last_undegraded":"2023-10-09T04:54:47.239521+0000","last_fullsized":"2023-10-09T04:54:47.239521+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":291,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240145+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:54:47.240145+0000","last_peered":"2023-10-09T04:54:47.240145+0000","last_clean":"2023-10-09T04:54:47.240145+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:54:47.240145+0000","last_undegraded":"2023-10-09T04:54:47.240145+0000","last_fullsized":"2023-10-09T04:54:47.240145+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"307'6452","reported_seq":6475,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.581598+0000","last_change":"2023-10-09T04:53:24.127013+0000","last_active":"2023-10-09T04:55:12.581598+0000","last_peered":"2023-10-09T04:55:12.581598+0000","last_clean":"2023-10-09T04:55:12.581598+0000","last_became_active":"2023-10-09T04:53:24.126651+0000","last_became_peered":"2023-10-09T04:53:24.126651+0000","last_unstale":"2023-10-09T04:55:12.581598+0000","last_undegraded":"2023-10-09T04:55:12.581598+0000","last_fullsized":"2023-10-09T04:55:12.581598+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:33.048421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422838272,"num_objects":6452,"num_object_clones":0,"num_object_copies":19356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6452,"num_write_kb":412928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":240,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231953+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:54:47.231953+0000","last_peered":"2023-10-09T04:54:47.231953+0000","last_clean":"2023-10-09T04:54:47.231953+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:54:47.231953+0000","last_undegraded":"2023-10-09T04:54:47.231953+0000","last_fullsized":"2023-10-09T04:54:47.231953+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247667+0000","last_change":"2023-10-09T04:51:14.146161+0000","last_active":"2023-10-09T04:54:47.247667+0000","last_peered":"2023-10-09T04:54:47.247667+0000","last_clean":"2023-10-09T04:54:47.247667+0000","last_became_active":"2023-10-09T04:51:14.145388+0000","last_became_peered":"2023-10-09T04:51:14.145388+0000","last_unstale":"2023-10-09T04:54:47.247667+0000","last_undegraded":"2023-10-09T04:54:47.247667+0000","last_fullsized":"2023-10-09T04:54:47.247667+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:03.104400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"307'6374","reported_seq":6397,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.073602+0000","last_change":"2023-10-09T04:53:24.350004+0000","last_active":"2023-10-09T04:55:12.073602+0000","last_peered":"2023-10-09T04:55:12.073602+0000","last_clean":"2023-10-09T04:55:12.073602+0000","last_became_active":"2023-10-09T04:53:24.349539+0000","last_became_peered":"2023-10-09T04:53:24.349539+0000","last_unstale":"2023-10-09T04:55:12.073602+0000","last_undegraded":"2023-10-09T04:55:12.073602+0000","last_fullsized":"2023-10-09T04:55:12.073602+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6374,"log_dups_size":0,"ondisk_log_size":6374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:55.535031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417726464,"num_objects":6374,"num_object_clones":0,"num_object_copies":19122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6374,"num_write_kb":407936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":243,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247853+0000","last_change":"2023-10-09T04:51:14.181439+0000","last_active":"2023-10-09T04:54:47.247853+0000","last_peered":"2023-10-09T04:54:47.247853+0000","last_clean":"2023-10-09T04:54:47.247853+0000","last_became_active":"2023-10-09T04:51:14.150700+0000","last_became_peered":"2023-10-09T04:51:14.150700+0000","last_unstale":"2023-10-09T04:54:47.247853+0000","last_undegraded":"2023-10-09T04:54:47.247853+0000","last_fullsized":"2023-10-09T04:54:47.247853+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:26.754849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240641+0000","last_change":"2023-10-09T04:51:14.151318+0000","last_active":"2023-10-09T04:54:47.240641+0000","last_peered":"2023-10-09T04:54:47.240641+0000","last_clean":"2023-10-09T04:54:47.240641+0000","last_became_active":"2023-10-09T04:51:14.150870+0000","last_became_peered":"2023-10-09T04:51:14.150870+0000","last_unstale":"2023-10-09T04:54:47.240641+0000","last_undegraded":"2023-10-09T04:54:47.240641+0000","last_fullsized":"2023-10-09T04:54:47.240641+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"307'6416","reported_seq":6439,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.595809+0000","last_change":"2023-10-09T04:53:24.136245+0000","last_active":"2023-10-09T04:55:12.595809+0000","last_peered":"2023-10-09T04:55:12.595809+0000","last_clean":"2023-10-09T04:55:12.595809+0000","last_became_active":"2023-10-09T04:53:24.135900+0000","last_became_peered":"2023-10-09T04:53:24.135900+0000","last_unstale":"2023-10-09T04:55:12.595809+0000","last_undegraded":"2023-10-09T04:55:12.595809+0000","last_fullsized":"2023-10-09T04:55:12.595809+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:27.074775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420478976,"num_objects":6416,"num_object_clones":0,"num_object_copies":19248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6416,"num_write_kb":410624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240035+0000","last_change":"2023-10-09T04:51:14.149198+0000","last_active":"2023-10-09T04:54:47.240035+0000","last_peered":"2023-10-09T04:54:47.240035+0000","last_clean":"2023-10-09T04:54:47.240035+0000","last_became_active":"2023-10-09T04:51:14.148826+0000","last_became_peered":"2023-10-09T04:51:14.148826+0000","last_unstale":"2023-10-09T04:54:47.240035+0000","last_undegraded":"2023-10-09T04:54:47.240035+0000","last_fullsized":"2023-10-09T04:54:47.240035+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":190,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247204+0000","last_change":"2023-10-09T04:51:14.109636+0000","last_active":"2023-10-09T04:54:47.247204+0000","last_peered":"2023-10-09T04:54:47.247204+0000","last_clean":"2023-10-09T04:54:47.247204+0000","last_became_active":"2023-10-09T04:51:14.109258+0000","last_became_peered":"2023-10-09T04:51:14.109258+0000","last_unstale":"2023-10-09T04:54:47.247204+0000","last_undegraded":"2023-10-09T04:54:47.247204+0000","last_fullsized":"2023-10-09T04:54:47.247204+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.321575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"307'6214","reported_seq":6236,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.654216+0000","last_change":"2023-10-09T04:53:24.349881+0000","last_active":"2023-10-09T04:55:08.654216+0000","last_peered":"2023-10-09T04:55:08.654216+0000","last_clean":"2023-10-09T04:55:08.654216+0000","last_became_active":"2023-10-09T04:53:24.348953+0000","last_became_peered":"2023-10-09T04:53:24.348953+0000","last_unstale":"2023-10-09T04:55:08.654216+0000","last_undegraded":"2023-10-09T04:55:08.654216+0000","last_fullsized":"2023-10-09T04:55:08.654216+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407240704,"num_objects":6214,"num_object_clones":0,"num_object_copies":18642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6214,"num_write_kb":397696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":140,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.246802+0000","last_change":"2023-10-09T04:51:14.125565+0000","last_active":"2023-10-09T04:54:47.246802+0000","last_peered":"2023-10-09T04:54:47.246802+0000","last_clean":"2023-10-09T04:54:47.246802+0000","last_became_active":"2023-10-09T04:51:14.125345+0000","last_became_peered":"2023-10-09T04:51:14.125345+0000","last_unstale":"2023-10-09T04:54:47.246802+0000","last_undegraded":"2023-10-09T04:54:47.246802+0000","last_fullsized":"2023-10-09T04:54:47.246802+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:14.172443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240701+0000","last_change":"2023-10-09T04:51:14.146838+0000","last_active":"2023-10-09T04:54:47.240701+0000","last_peered":"2023-10-09T04:54:47.240701+0000","last_clean":"2023-10-09T04:54:47.240701+0000","last_became_active":"2023-10-09T04:51:14.146457+0000","last_became_peered":"2023-10-09T04:51:14.146457+0000","last_unstale":"2023-10-09T04:54:47.240701+0000","last_undegraded":"2023-10-09T04:54:47.240701+0000","last_fullsized":"2023-10-09T04:54:47.240701+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"307'6458","reported_seq":6480,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:11.399654+0000","last_change":"2023-10-09T04:53:24.350320+0000","last_active":"2023-10-09T04:55:11.399654+0000","last_peered":"2023-10-09T04:55:11.399654+0000","last_clean":"2023-10-09T04:55:11.399654+0000","last_became_active":"2023-10-09T04:53:24.349443+0000","last_became_peered":"2023-10-09T04:53:24.349443+0000","last_unstale":"2023-10-09T04:55:11.399654+0000","last_undegraded":"2023-10-09T04:55:11.399654+0000","last_fullsized":"2023-10-09T04:55:11.399654+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423231488,"num_objects":6458,"num_object_clones":0,"num_object_copies":19374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6458,"num_write_kb":413312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":284,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229393+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:54:47.229393+0000","last_peered":"2023-10-09T04:54:47.229393+0000","last_clean":"2023-10-09T04:54:47.229393+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:54:47.229393+0000","last_undegraded":"2023-10-09T04:54:47.229393+0000","last_fullsized":"2023-10-09T04:54:47.229393+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":37,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239875+0000","last_change":"2023-10-09T04:51:14.155214+0000","last_active":"2023-10-09T04:54:47.239875+0000","last_peered":"2023-10-09T04:54:47.239875+0000","last_clean":"2023-10-09T04:54:47.239875+0000","last_became_active":"2023-10-09T04:51:14.154876+0000","last_became_peered":"2023-10-09T04:51:14.154876+0000","last_unstale":"2023-10-09T04:54:47.239875+0000","last_undegraded":"2023-10-09T04:54:47.239875+0000","last_fullsized":"2023-10-09T04:54:47.239875+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.c","version":"307'6310","reported_seq":6332,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:11.441931+0000","last_change":"2023-10-09T04:53:24.135648+0000","last_active":"2023-10-09T04:55:11.441931+0000","last_peered":"2023-10-09T04:55:11.441931+0000","last_clean":"2023-10-09T04:55:11.441931+0000","last_became_active":"2023-10-09T04:53:24.135052+0000","last_became_peered":"2023-10-09T04:53:24.135052+0000","last_unstale":"2023-10-09T04:55:11.441931+0000","last_undegraded":"2023-10-09T04:55:11.441931+0000","last_fullsized":"2023-10-09T04:55:11.441931+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413532160,"num_objects":6310,"num_object_clones":0,"num_object_copies":18930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6310,"num_write_kb":403840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":472,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:57.065995+0000","last_change":"2023-10-09T04:50:44.050055+0000","last_active":"2023-10-09T04:54:57.065995+0000","last_peered":"2023-10-09T04:54:57.065995+0000","last_clean":"2023-10-09T04:54:57.065995+0000","last_became_active":"2023-10-09T04:50:44.049620+0000","last_became_peered":"2023-10-09T04:50:44.049620+0000","last_unstale":"2023-10-09T04:54:57.065995+0000","last_undegraded":"2023-10-09T04:54:57.065995+0000","last_fullsized":"2023-10-09T04:54:57.065995+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.943483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":270,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232252+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:54:47.232252+0000","last_peered":"2023-10-09T04:54:47.232252+0000","last_clean":"2023-10-09T04:54:47.232252+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:54:47.232252+0000","last_undegraded":"2023-10-09T04:54:47.232252+0000","last_fullsized":"2023-10-09T04:54:47.232252+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":241,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231999+0000","last_change":"2023-10-09T04:51:14.108030+0000","last_active":"2023-10-09T04:54:47.231999+0000","last_peered":"2023-10-09T04:54:47.231999+0000","last_clean":"2023-10-09T04:54:47.231999+0000","last_became_active":"2023-10-09T04:51:14.107660+0000","last_became_peered":"2023-10-09T04:51:14.107660+0000","last_unstale":"2023-10-09T04:54:47.231999+0000","last_undegraded":"2023-10-09T04:54:47.231999+0000","last_fullsized":"2023-10-09T04:54:47.231999+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"307'6257","reported_seq":6277,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.660976+0000","last_change":"2023-10-09T04:53:24.137716+0000","last_active":"2023-10-09T04:55:08.660976+0000","last_peered":"2023-10-09T04:55:08.660976+0000","last_clean":"2023-10-09T04:55:08.660976+0000","last_became_active":"2023-10-09T04:53:24.137204+0000","last_became_peered":"2023-10-09T04:53:24.137204+0000","last_unstale":"2023-10-09T04:55:08.660976+0000","last_undegraded":"2023-10-09T04:55:08.660976+0000","last_fullsized":"2023-10-09T04:55:08.660976+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6257,"log_dups_size":0,"ondisk_log_size":6257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410058752,"num_objects":6257,"num_object_clones":0,"num_object_copies":18771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6257,"num_write_kb":400448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":769,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231963+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:54:47.231963+0000","last_peered":"2023-10-09T04:54:47.231963+0000","last_clean":"2023-10-09T04:54:47.231963+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:54:47.231963+0000","last_undegraded":"2023-10-09T04:54:47.231963+0000","last_fullsized":"2023-10-09T04:54:47.231963+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":157,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248013+0000","last_change":"2023-10-09T04:51:14.112695+0000","last_active":"2023-10-09T04:54:47.248013+0000","last_peered":"2023-10-09T04:54:47.248013+0000","last_clean":"2023-10-09T04:54:47.248013+0000","last_became_active":"2023-10-09T04:51:14.112434+0000","last_became_peered":"2023-10-09T04:51:14.112434+0000","last_unstale":"2023-10-09T04:54:47.248013+0000","last_undegraded":"2023-10-09T04:54:47.248013+0000","last_fullsized":"2023-10-09T04:54:47.248013+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:10.034798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":253,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:57.604552+0000","last_change":"2023-10-09T04:50:44.062081+0000","last_active":"2023-10-09T04:54:57.604552+0000","last_peered":"2023-10-09T04:54:57.604552+0000","last_clean":"2023-10-09T04:54:57.604552+0000","last_became_active":"2023-10-09T04:50:44.061730+0000","last_became_peered":"2023-10-09T04:50:44.061730+0000","last_unstale":"2023-10-09T04:54:57.604552+0000","last_undegraded":"2023-10-09T04:54:57.604552+0000","last_fullsized":"2023-10-09T04:54:57.604552+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:52.943977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"307'6250","reported_seq":6272,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:09.956475+0000","last_change":"2023-10-09T04:53:24.136930+0000","last_active":"2023-10-09T04:55:09.956475+0000","last_peered":"2023-10-09T04:55:09.956475+0000","last_clean":"2023-10-09T04:55:09.956475+0000","last_became_active":"2023-10-09T04:53:24.136546+0000","last_became_peered":"2023-10-09T04:53:24.136546+0000","last_unstale":"2023-10-09T04:55:09.956475+0000","last_undegraded":"2023-10-09T04:55:09.956475+0000","last_fullsized":"2023-10-09T04:55:09.956475+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6250,"log_dups_size":0,"ondisk_log_size":6250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409600000,"num_objects":6250,"num_object_clones":0,"num_object_copies":18750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6250,"num_write_kb":400000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1388,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201007+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:55:08.201007+0000","last_peered":"2023-10-09T04:55:08.201007+0000","last_clean":"2023-10-09T04:55:08.201007+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:55:08.201007+0000","last_undegraded":"2023-10-09T04:55:08.201007+0000","last_fullsized":"2023-10-09T04:55:08.201007+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.593170+0000","last_change":"2023-10-09T04:50:44.047494+0000","last_active":"2023-10-09T04:54:47.593170+0000","last_peered":"2023-10-09T04:54:47.593170+0000","last_clean":"2023-10-09T04:54:47.593170+0000","last_became_active":"2023-10-09T04:50:44.047111+0000","last_became_peered":"2023-10-09T04:50:44.047111+0000","last_unstale":"2023-10-09T04:54:47.593170+0000","last_undegraded":"2023-10-09T04:54:47.593170+0000","last_fullsized":"2023-10-09T04:54:47.593170+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:49.698697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232499+0000","last_change":"2023-10-09T04:50:44.051844+0000","last_active":"2023-10-09T04:54:47.232499+0000","last_peered":"2023-10-09T04:54:47.232499+0000","last_clean":"2023-10-09T04:54:47.232499+0000","last_became_active":"2023-10-09T04:50:44.051376+0000","last_became_peered":"2023-10-09T04:50:44.051376+0000","last_unstale":"2023-10-09T04:54:47.232499+0000","last_undegraded":"2023-10-09T04:54:47.232499+0000","last_fullsized":"2023-10-09T04:54:47.232499+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"307'6207","reported_seq":6229,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.592985+0000","last_change":"2023-10-09T04:53:24.352741+0000","last_active":"2023-10-09T04:55:08.592985+0000","last_peered":"2023-10-09T04:55:08.592985+0000","last_clean":"2023-10-09T04:55:08.592985+0000","last_became_active":"2023-10-09T04:53:24.351966+0000","last_became_peered":"2023-10-09T04:53:24.351966+0000","last_unstale":"2023-10-09T04:55:08.592985+0000","last_undegraded":"2023-10-09T04:55:08.592985+0000","last_fullsized":"2023-10-09T04:55:08.592985+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406781952,"num_objects":6207,"num_object_clones":0,"num_object_copies":18621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6207,"num_write_kb":397248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":832,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232076+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:54:47.232076+0000","last_peered":"2023-10-09T04:54:47.232076+0000","last_clean":"2023-10-09T04:54:47.232076+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:54:47.232076+0000","last_undegraded":"2023-10-09T04:54:47.232076+0000","last_fullsized":"2023-10-09T04:54:47.232076+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":528,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.241210+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:54:47.241210+0000","last_peered":"2023-10-09T04:54:47.241210+0000","last_clean":"2023-10-09T04:54:47.241210+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:54:47.241210+0000","last_undegraded":"2023-10-09T04:54:47.241210+0000","last_fullsized":"2023-10-09T04:54:47.241210+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":225,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231865+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:54:47.231865+0000","last_peered":"2023-10-09T04:54:47.231865+0000","last_clean":"2023-10-09T04:54:47.231865+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:54:47.231865+0000","last_undegraded":"2023-10-09T04:54:47.231865+0000","last_fullsized":"2023-10-09T04:54:47.231865+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":250,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.241135+0000","last_change":"2023-10-09T04:51:14.125228+0000","last_active":"2023-10-09T04:54:47.241135+0000","last_peered":"2023-10-09T04:54:47.241135+0000","last_clean":"2023-10-09T04:54:47.241135+0000","last_became_active":"2023-10-09T04:51:14.124848+0000","last_became_peered":"2023-10-09T04:51:14.124848+0000","last_unstale":"2023-10-09T04:54:47.241135+0000","last_undegraded":"2023-10-09T04:54:47.241135+0000","last_fullsized":"2023-10-09T04:54:47.241135+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"307'6315","reported_seq":6338,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:11.959626+0000","last_change":"2023-10-09T04:53:24.136377+0000","last_active":"2023-10-09T04:55:11.959626+0000","last_peered":"2023-10-09T04:55:11.959626+0000","last_clean":"2023-10-09T04:55:11.959626+0000","last_became_active":"2023-10-09T04:53:24.136061+0000","last_became_peered":"2023-10-09T04:53:24.136061+0000","last_unstale":"2023-10-09T04:55:11.959626+0000","last_undegraded":"2023-10-09T04:55:11.959626+0000","last_fullsized":"2023-10-09T04:55:11.959626+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6315,"log_dups_size":0,"ondisk_log_size":6315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:09.217233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413859840,"num_objects":6315,"num_object_clones":0,"num_object_copies":18945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6315,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":694,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.241034+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:54:47.241034+0000","last_peered":"2023-10-09T04:54:47.241034+0000","last_clean":"2023-10-09T04:54:47.241034+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:54:47.241034+0000","last_undegraded":"2023-10-09T04:54:47.241034+0000","last_fullsized":"2023-10-09T04:54:47.241034+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":324,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.244125+0000","last_change":"2023-10-09T04:51:14.151803+0000","last_active":"2023-10-09T04:54:47.244125+0000","last_peered":"2023-10-09T04:54:47.244125+0000","last_clean":"2023-10-09T04:54:47.244125+0000","last_became_active":"2023-10-09T04:51:14.151334+0000","last_became_peered":"2023-10-09T04:51:14.151334+0000","last_unstale":"2023-10-09T04:54:47.244125+0000","last_undegraded":"2023-10-09T04:54:47.244125+0000","last_fullsized":"2023-10-09T04:54:47.244125+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:21.885966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":298,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.059881+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:54:52.059881+0000","last_peered":"2023-10-09T04:54:52.059881+0000","last_clean":"2023-10-09T04:54:52.059881+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:54:52.059881+0000","last_undegraded":"2023-10-09T04:54:52.059881+0000","last_fullsized":"2023-10-09T04:54:52.059881+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"307'6148","reported_seq":6170,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.643348+0000","last_change":"2023-10-09T04:53:24.140748+0000","last_active":"2023-10-09T04:55:08.643348+0000","last_peered":"2023-10-09T04:55:08.643348+0000","last_clean":"2023-10-09T04:55:08.643348+0000","last_became_active":"2023-10-09T04:53:24.140325+0000","last_became_peered":"2023-10-09T04:53:24.140325+0000","last_unstale":"2023-10-09T04:55:08.643348+0000","last_undegraded":"2023-10-09T04:55:08.643348+0000","last_fullsized":"2023-10-09T04:55:08.643348+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6148,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":565,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239588+0000","last_change":"2023-10-09T04:53:05.183235+0000","last_active":"2023-10-09T04:54:47.239588+0000","last_peered":"2023-10-09T04:54:47.239588+0000","last_clean":"2023-10-09T04:54:47.239588+0000","last_became_active":"2023-10-09T04:53:05.182472+0000","last_became_peered":"2023-10-09T04:53:05.182472+0000","last_unstale":"2023-10-09T04:54:47.239588+0000","last_undegraded":"2023-10-09T04:54:47.239588+0000","last_fullsized":"2023-10-09T04:54:47.239588+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":253,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:57.604574+0000","last_change":"2023-10-09T04:50:44.052455+0000","last_active":"2023-10-09T04:54:57.604574+0000","last_peered":"2023-10-09T04:54:57.604574+0000","last_clean":"2023-10-09T04:54:57.604574+0000","last_became_active":"2023-10-09T04:50:44.052136+0000","last_became_peered":"2023-10-09T04:50:44.052136+0000","last_unstale":"2023-10-09T04:54:57.604574+0000","last_undegraded":"2023-10-09T04:54:57.604574+0000","last_fullsized":"2023-10-09T04:54:57.604574+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:32.549357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":286,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.247963+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:54:47.247963+0000","last_peered":"2023-10-09T04:54:47.247963+0000","last_clean":"2023-10-09T04:54:47.247963+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:54:47.247963+0000","last_undegraded":"2023-10-09T04:54:47.247963+0000","last_fullsized":"2023-10-09T04:54:47.247963+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"307'6075","reported_seq":6094,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.663766+0000","last_change":"2023-10-09T04:53:24.137672+0000","last_active":"2023-10-09T04:55:08.663766+0000","last_peered":"2023-10-09T04:55:08.663766+0000","last_clean":"2023-10-09T04:55:08.663766+0000","last_became_active":"2023-10-09T04:53:24.137340+0000","last_became_peered":"2023-10-09T04:53:24.137340+0000","last_unstale":"2023-10-09T04:55:08.663766+0000","last_undegraded":"2023-10-09T04:55:08.663766+0000","last_fullsized":"2023-10-09T04:55:08.663766+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398131200,"num_objects":6075,"num_object_clones":0,"num_object_copies":18225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6075,"num_write_kb":388800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":126,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239900+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:54:47.239900+0000","last_peered":"2023-10-09T04:54:47.239900+0000","last_clean":"2023-10-09T04:54:47.239900+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:54:47.239900+0000","last_undegraded":"2023-10-09T04:54:47.239900+0000","last_fullsized":"2023-10-09T04:54:47.239900+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":367,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240875+0000","last_change":"2023-10-09T04:51:14.350042+0000","last_active":"2023-10-09T04:54:47.240875+0000","last_peered":"2023-10-09T04:54:47.240875+0000","last_clean":"2023-10-09T04:54:47.240875+0000","last_became_active":"2023-10-09T04:51:14.157185+0000","last_became_peered":"2023-10-09T04:51:14.157185+0000","last_unstale":"2023-10-09T04:54:47.240875+0000","last_undegraded":"2023-10-09T04:54:47.240875+0000","last_fullsized":"2023-10-09T04:54:47.240875+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.239857+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:54:47.239857+0000","last_peered":"2023-10-09T04:54:47.239857+0000","last_clean":"2023-10-09T04:54:47.239857+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:54:47.239857+0000","last_undegraded":"2023-10-09T04:54:47.239857+0000","last_fullsized":"2023-10-09T04:54:47.239857+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"307'6297","reported_seq":6319,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.643468+0000","last_change":"2023-10-09T04:53:24.140212+0000","last_active":"2023-10-09T04:55:08.643468+0000","last_peered":"2023-10-09T04:55:08.643468+0000","last_clean":"2023-10-09T04:55:08.643468+0000","last_became_active":"2023-10-09T04:53:24.139824+0000","last_became_peered":"2023-10-09T04:53:24.139824+0000","last_unstale":"2023-10-09T04:55:08.643468+0000","last_undegraded":"2023-10-09T04:55:08.643468+0000","last_fullsized":"2023-10-09T04:55:08.643468+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6297,"log_dups_size":0,"ondisk_log_size":6297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412680192,"num_objects":6297,"num_object_clones":0,"num_object_copies":18891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6297,"num_write_kb":403008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":387,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.241308+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:54:47.241308+0000","last_peered":"2023-10-09T04:54:47.241308+0000","last_clean":"2023-10-09T04:54:47.241308+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:54:47.241308+0000","last_undegraded":"2023-10-09T04:54:47.241308+0000","last_fullsized":"2023-10-09T04:54:47.241308+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":285,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.059904+0000","last_change":"2023-10-09T04:51:14.178205+0000","last_active":"2023-10-09T04:54:52.059904+0000","last_peered":"2023-10-09T04:54:52.059904+0000","last_clean":"2023-10-09T04:54:52.059904+0000","last_became_active":"2023-10-09T04:51:14.147035+0000","last_became_peered":"2023-10-09T04:51:14.147035+0000","last_unstale":"2023-10-09T04:54:52.059904+0000","last_undegraded":"2023-10-09T04:54:52.059904+0000","last_fullsized":"2023-10-09T04:54:52.059904+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:04.219571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231946+0000","last_change":"2023-10-09T04:51:14.110047+0000","last_active":"2023-10-09T04:54:47.231946+0000","last_peered":"2023-10-09T04:54:47.231946+0000","last_clean":"2023-10-09T04:54:47.231946+0000","last_became_active":"2023-10-09T04:51:14.109658+0000","last_became_peered":"2023-10-09T04:51:14.109658+0000","last_unstale":"2023-10-09T04:54:47.231946+0000","last_undegraded":"2023-10-09T04:54:47.231946+0000","last_fullsized":"2023-10-09T04:54:47.231946+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":282,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.059812+0000","last_change":"2023-10-09T04:51:14.148515+0000","last_active":"2023-10-09T04:54:52.059812+0000","last_peered":"2023-10-09T04:54:52.059812+0000","last_clean":"2023-10-09T04:54:52.059812+0000","last_became_active":"2023-10-09T04:51:14.147834+0000","last_became_peered":"2023-10-09T04:51:14.147834+0000","last_unstale":"2023-10-09T04:54:52.059812+0000","last_undegraded":"2023-10-09T04:54:52.059812+0000","last_fullsized":"2023-10-09T04:54:52.059812+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:48.693686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":254,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:52.597421+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:54:52.597421+0000","last_peered":"2023-10-09T04:54:52.597421+0000","last_clean":"2023-10-09T04:54:52.597421+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:54:52.597421+0000","last_undegraded":"2023-10-09T04:54:52.597421+0000","last_fullsized":"2023-10-09T04:54:52.597421+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":245,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232073+0000","last_change":"2023-10-09T04:51:14.120655+0000","last_active":"2023-10-09T04:54:47.232073+0000","last_peered":"2023-10-09T04:54:47.232073+0000","last_clean":"2023-10-09T04:54:47.232073+0000","last_became_active":"2023-10-09T04:51:14.109662+0000","last_became_peered":"2023-10-09T04:51:14.109662+0000","last_unstale":"2023-10-09T04:54:47.232073+0000","last_undegraded":"2023-10-09T04:54:47.232073+0000","last_fullsized":"2023-10-09T04:54:47.232073+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":39,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.246949+0000","last_change":"2023-10-09T04:51:14.152593+0000","last_active":"2023-10-09T04:54:47.246949+0000","last_peered":"2023-10-09T04:54:47.246949+0000","last_clean":"2023-10-09T04:54:47.246949+0000","last_became_active":"2023-10-09T04:51:14.152155+0000","last_became_peered":"2023-10-09T04:51:14.152155+0000","last_unstale":"2023-10-09T04:54:47.246949+0000","last_undegraded":"2023-10-09T04:54:47.246949+0000","last_fullsized":"2023-10-09T04:54:47.246949+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:51.059337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":238,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232251+0000","last_change":"2023-10-09T04:51:14.164283+0000","last_active":"2023-10-09T04:54:47.232251+0000","last_peered":"2023-10-09T04:54:47.232251+0000","last_clean":"2023-10-09T04:54:47.232251+0000","last_became_active":"2023-10-09T04:51:14.111363+0000","last_became_peered":"2023-10-09T04:51:14.111363+0000","last_unstale":"2023-10-09T04:54:47.232251+0000","last_undegraded":"2023-10-09T04:54:47.232251+0000","last_fullsized":"2023-10-09T04:54:47.232251+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":230,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.245019+0000","last_change":"2023-10-09T04:51:14.150509+0000","last_active":"2023-10-09T04:54:47.245019+0000","last_peered":"2023-10-09T04:54:47.245019+0000","last_clean":"2023-10-09T04:54:47.245019+0000","last_became_active":"2023-10-09T04:51:14.150128+0000","last_became_peered":"2023-10-09T04:51:14.150128+0000","last_unstale":"2023-10-09T04:54:47.245019+0000","last_undegraded":"2023-10-09T04:54:47.245019+0000","last_fullsized":"2023-10-09T04:54:47.245019+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:17.197700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":208,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240688+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:54:47.240688+0000","last_peered":"2023-10-09T04:54:47.240688+0000","last_clean":"2023-10-09T04:54:47.240688+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:54:47.240688+0000","last_undegraded":"2023-10-09T04:54:47.240688+0000","last_fullsized":"2023-10-09T04:54:47.240688+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231828+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T04:54:47.231828+0000","last_peered":"2023-10-09T04:54:47.231828+0000","last_clean":"2023-10-09T04:54:47.231828+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T04:54:47.231828+0000","last_undegraded":"2023-10-09T04:54:47.231828+0000","last_fullsized":"2023-10-09T04:54:47.231828+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":302,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.248059+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:54:47.248059+0000","last_peered":"2023-10-09T04:54:47.248059+0000","last_clean":"2023-10-09T04:54:47.248059+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:54:47.248059+0000","last_undegraded":"2023-10-09T04:54:47.248059+0000","last_fullsized":"2023-10-09T04:54:47.248059+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.240088+0000","last_change":"2023-10-09T04:51:14.146277+0000","last_active":"2023-10-09T04:54:47.240088+0000","last_peered":"2023-10-09T04:54:47.240088+0000","last_clean":"2023-10-09T04:54:47.240088+0000","last_became_active":"2023-10-09T04:51:14.145974+0000","last_became_peered":"2023-10-09T04:51:14.145974+0000","last_unstale":"2023-10-09T04:54:47.240088+0000","last_undegraded":"2023-10-09T04:54:47.240088+0000","last_fullsized":"2023-10-09T04:54:47.240088+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6599409664,"num_objects":100699,"num_object_clones":0,"num_object_copies":302097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100699,"num_write_kb":6444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19810222080,"data_stored":19810222080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100699,"ondisk_log_size":100699,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":9830,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430769,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2932196,"kb_used_data":2032668,"kb_used_omap":0,"kb_used_meta":899520,"kb_avail":90812956,"statfs":{"total":95995035648,"available":92992466944,"internally_reserved":0,"allocated":2081452032,"data_stored":2080847833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921108480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:27 2023","interfaces":[{"interface":"back","average":{"1min":30.649000000000001,"5min":9.5790000000000006,"15min":4.8520000000000003},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":422.35000000000002,"5min":422.35000000000002,"15min":422.35000000000002},"last":1.4650000000000001},{"interface":"front","average":{"1min":32.039000000000001,"5min":9.827,"15min":4.8259999999999996},"min":{"1min":0.38600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":423.267,"5min":423.267,"15min":423.267},"last":1.5800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":2.8319999999999999,"15min":2.3050000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":46.222999999999999,"5min":57.668999999999997,"15min":58.816000000000003},"last":1.353},{"interface":"front","average":{"1min":3.016,"5min":2.895,"15min":2.2589999999999999},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":45.536000000000001,"5min":58.046999999999997,"15min":58.840000000000003},"last":4.1799999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.6850000000000001,"15min":1.6519999999999999},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.746,"5min":13.138999999999999,"15min":13.138999999999999},"last":3.722},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.3260000000000001,"15min":1.944},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.7789999999999999,"5min":7.4530000000000003,"15min":11.583},"last":0.33800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.6100000000000001,"15min":2.069},"min":{"1min":0.42399999999999999,"5min":0.374,"15min":0.26700000000000002},"max":{"1min":2.254,"5min":8.6449999999999996,"15min":102.02800000000001},"last":1.774},{"interface":"front","average":{"1min":0.752,"5min":1.7030000000000001,"15min":2.0230000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.32100000000000001},"max":{"1min":2.101,"5min":13.195,"15min":101.973},"last":1.7609999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.294,"15min":1.3029999999999999},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.012,"5min":10.199,"15min":11.933999999999999},"last":0.377},{"interface":"front","average":{"1min":0.77600000000000002,"5min":1.2470000000000001,"15min":1.341},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.308},"max":{"1min":2.2930000000000001,"5min":8.1080000000000005,"15min":8.452},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:54:16 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.7390000000000001,"15min":1.496},"min":{"1min":0.41399999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.397,"5min":8.6050000000000004,"15min":11.268000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.9119999999999999,"15min":1.6279999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":2.4300000000000002,"5min":9.1449999999999996,"15min":15.138999999999999},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.173,"5min":1.6899999999999999,"15min":1.472},"min":{"1min":0.442,"5min":0.40400000000000003,"15min":0.36899999999999999},"max":{"1min":27.059000000000001,"5min":27.059000000000001,"15min":27.059000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.77,"15min":1.552},"min":{"1min":0.44,"5min":0.39300000000000002,"15min":0.36599999999999999},"max":{"1min":27.698,"5min":27.698,"15min":27.698},"last":0.52400000000000002}]}]},{"osd":6,"up_from":146,"seq":627065225469,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3388760,"kb_used_data":2762128,"kb_used_omap":5,"kb_used_meta":626618,"kb_avail":90356392,"statfs":{"total":95995035648,"available":92524945408,"internally_reserved":0,"allocated":2828419072,"data_stored":2827819228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":641657440},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":6.4429999999999996,"5min":2.996,"15min":3.4580000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":81.287999999999997,"5min":81.287999999999997,"15min":214.82300000000001},"last":3.1680000000000001},{"interface":"front","average":{"1min":6.2370000000000001,"5min":2.4689999999999999,"15min":3.2799999999999998},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":81.168999999999997,"5min":81.168999999999997,"15min":213.34700000000001},"last":3.056}]},{"osd":1,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.5409999999999999,"15min":2.6469999999999998},"min":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":9.9570000000000007,"5min":40.709000000000003,"15min":76.081000000000003},"last":2.698},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.5369999999999999,"15min":2.3900000000000001},"min":{"1min":0.39400000000000002,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":5.742,"5min":40.414000000000001,"15min":75.546000000000006},"last":2.879}]},{"osd":2,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":1.2709999999999999,"15min":1.607},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6890000000000001,"5min":5.8540000000000001,"15min":6.8940000000000001},"last":1.486},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97099999999999997,"15min":1.2769999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.311},"max":{"1min":4.6669999999999998,"5min":5.0709999999999997,"15min":9.4290000000000003},"last":1.8640000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":1.6240000000000001,"15min":1.5840000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":12.050000000000001,"5min":12.050000000000001,"15min":12.050000000000001},"last":1.512},{"interface":"front","average":{"1min":1.883,"5min":1.5960000000000001,"15min":1.48},"min":{"1min":0.41999999999999998,"5min":0.34499999999999997,"15min":0.32600000000000001},"max":{"1min":5.1200000000000001,"5min":8.9079999999999995,"15min":9.3190000000000008},"last":1.526}]},{"osd":4,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.88,"15min":1.1160000000000001},"min":{"1min":0.38900000000000001,"5min":0.32800000000000001,"15min":0.248},"max":{"1min":4.7069999999999999,"5min":4.7069999999999999,"15min":13.699999999999999},"last":0.317},{"interface":"front","average":{"1min":1.657,"5min":1.3779999999999999,"15min":1.3939999999999999},"min":{"1min":0.36599999999999999,"5min":0.35899999999999999,"15min":0.307},"max":{"1min":7.476,"5min":10.428000000000001,"15min":10.428000000000001},"last":1.397}]},{"osd":5,"last update":"Mon Oct 9 04:54:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.125,"15min":1.2929999999999999},"min":{"1min":0.42299999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":1.7889999999999999,"5min":10.871,"15min":10.871},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.984,"15min":1.675},"min":{"1min":0.44800000000000001,"5min":0.40200000000000002,"15min":0.29599999999999999},"max":{"1min":2.778,"5min":11.321999999999999,"15min":11.321999999999999},"last":1.456}]},{"osd":7,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.6719999999999999,"15min":1.6180000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.33000000000000002},"max":{"1min":10.526999999999999,"5min":10.701000000000001,"15min":16.527999999999999},"last":1.337},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.4530000000000001,"15min":1.4730000000000001},"min":{"1min":0.45900000000000002,"5min":0.41699999999999998,"15min":0.33300000000000002},"max":{"1min":4.7939999999999996,"5min":6.6180000000000003,"15min":15.366},"last":0.34200000000000003}]}]},{"osd":1,"up_from":93,"seq":399431958918,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3231040,"kb_used_data":2389880,"kb_used_omap":6,"kb_used_meta":841145,"kb_avail":90514112,"statfs":{"total":95995035648,"available":92686450688,"internally_reserved":0,"allocated":2447237120,"data_stored":2446084219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":861332492},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.2389999999999999,"15min":2.117},"min":{"1min":0.22800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":47.274000000000001,"5min":47.274000000000001,"15min":47.274000000000001},"last":0.501},{"interface":"front","average":{"1min":4.3600000000000003,"5min":1.8939999999999999,"15min":2.04},"min":{"1min":0.36799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":47.372999999999998,"5min":47.372999999999998,"15min":47.372999999999998},"last":0.52100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:55:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.361,"15min":1.9430000000000001},"min":{"1min":0.27600000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.1980000000000004,"5min":8.2650000000000006,"15min":15.336},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.194,"15min":1.7769999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.494,"5min":10.494,"15min":13.444000000000001},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:55:02 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":2.0030000000000001,"15min":1.827},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20999999999999999},"max":{"1min":10.589,"5min":40.210000000000001,"15min":40.210000000000001},"last":2.2650000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.0070000000000001,"15min":1.8},"min":{"1min":0.442,"5min":0.26000000000000001,"15min":0.20200000000000001},"max":{"1min":10.574,"5min":40.127000000000002,"15min":40.127000000000002},"last":2.3010000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.387,"15min":1.458},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":4.1769999999999996,"5min":7.923,"15min":13.362},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.752,"5min":1.627,"15min":1.599},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.316},"max":{"1min":9.6259999999999994,"5min":9.827,"15min":10.923999999999999},"last":0.57399999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:55:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.5580000000000001,"15min":1.649},"min":{"1min":0.374,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1039999999999992,"5min":9.2910000000000004,"15min":14.494999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.579,"15min":1.671},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":10.435,"5min":10.435,"15min":13.413},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2810000000000001,"5min":1.7549999999999999,"15min":1.7649999999999999},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.32600000000000001},"max":{"1min":18.295000000000002,"5min":18.295000000000002,"15min":18.295000000000002},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.7869999999999999,"15min":1.841},"min":{"1min":0.48699999999999999,"5min":0.41999999999999998,"15min":0.40500000000000003},"max":{"1min":18.34,"5min":18.34,"15min":18.34},"last":0.81499999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.5489999999999999,"15min":1.673},"min":{"1min":0.40699999999999997,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":4.1479999999999997,"5min":7.8899999999999997,"15min":13.474},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.623,"5min":1.889,"15min":1.7909999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.36399999999999999},"max":{"1min":9.5909999999999993,"5min":9.8079999999999998,"15min":18.074000000000002},"last":0.53300000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122444,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4938440,"kb_used_data":4068928,"kb_used_omap":9,"kb_used_meta":869494,"kb_avail":88806712,"statfs":{"total":95995035648,"available":90938073088,"internally_reserved":0,"allocated":4166582272,"data_stored":4165425765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":890362353},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,6,3,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":2.0680000000000001,"15min":1.585},"min":{"1min":0.254,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":5.2949999999999999,"5min":62.396000000000001,"15min":62.396000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.2280000000000002,"15min":2.0289999999999999},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19400000000000001},"max":{"1min":6.6180000000000003,"5min":62.496000000000002,"15min":62.496000000000002},"last":0.91500000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":1.1160000000000001,"15min":1.788},"min":{"1min":0.32300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.0830000000000002,"5min":8.0830000000000002,"15min":17.109000000000002},"last":1.889},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.365,"15min":2.1030000000000002},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.25600000000000001},"max":{"1min":6.2290000000000001,"5min":6.3929999999999998,"15min":12.57},"last":4.3289999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":1.4410000000000001,"15min":1.4490000000000001},"min":{"1min":0.35699999999999998,"5min":0.216,"15min":0.216},"max":{"1min":8.1150000000000002,"5min":8.1150000000000002,"15min":12.333},"last":1.784},{"interface":"front","average":{"1min":1.988,"5min":1.6899999999999999,"15min":1.792},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.23999999999999999},"max":{"1min":6.5720000000000001,"5min":8.1029999999999998,"15min":14.956},"last":1.75}]},{"osd":4,"last update":"Mon Oct 9 04:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.702,"15min":1.6479999999999999},"min":{"1min":0.56000000000000005,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":7.181,"5min":9.6509999999999998,"15min":16.84},"last":1.266},{"interface":"front","average":{"1min":1.149,"5min":1.157,"15min":1.3320000000000001},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6429999999999998,"5min":5.048,"15min":39.957999999999998},"last":1.2549999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.635,"15min":1.6499999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.35399999999999998},"max":{"1min":7.2889999999999997,"5min":7.2889999999999997,"15min":11.617000000000001},"last":1.325},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.671,"15min":1.5089999999999999},"min":{"1min":0.51300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.404,"5min":10.404,"15min":10.404},"last":1.1819999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.157,"15min":1.3779999999999999},"min":{"1min":0.45900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.9320000000000004,"5min":6.9320000000000004,"15min":10.345000000000001},"last":1.226},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.6519999999999999,"15min":1.7529999999999999},"min":{"1min":0.48699999999999999,"5min":0.41199999999999998,"15min":0.39400000000000002},"max":{"1min":7.7009999999999996,"5min":7.7009999999999996,"15min":11.57},"last":1.347}]},{"osd":7,"last update":"Mon Oct 9 04:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":2.1379999999999999,"15min":1.9379999999999999},"min":{"1min":0.69499999999999995,"5min":0.41699999999999998,"15min":0.38400000000000001},"max":{"1min":3.6549999999999998,"5min":43.276000000000003,"15min":43.276000000000003},"last":1.718},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.476,"15min":1.9630000000000001},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.2469999999999999,"5min":46.064,"15min":46.064},"last":1.24}]}]},{"osd":2,"up_from":234,"seq":1005022347389,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2731364,"kb_used_data":2447708,"kb_used_omap":13,"kb_used_meta":283634,"kb_avail":91013788,"statfs":{"total":95995035648,"available":93198118912,"internally_reserved":0,"allocated":2506452992,"data_stored":2505271426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":290442111},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":3.4769999999999999,"15min":2.806},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":6.0659999999999998,"5min":112.312,"15min":112.312},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.6910000000000001,"5min":3.5720000000000001,"15min":2.7999999999999998},"min":{"1min":0.245,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.7519999999999998,"5min":117.881,"15min":117.881},"last":0.89800000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":8.0589999999999993,"15min":4.6820000000000004},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.5890000000000004,"5min":267.54000000000002,"15min":267.54000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.659,"5min":8.1669999999999998,"15min":4.4249999999999998},"min":{"1min":0.27300000000000002,"5min":0.252,"15min":0.22900000000000001},"max":{"1min":5.8200000000000003,"5min":268.12200000000001,"15min":268.12200000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.474,"15min":1.4950000000000001},"min":{"1min":0.251,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.747999999999999,"5min":14.334,"15min":14.340999999999999},"last":3.4900000000000002},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.569,"15min":2.0499999999999998},"min":{"1min":0.34000000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.532,"5min":13.167999999999999,"15min":13.167999999999999},"last":2.4049999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.2370000000000001,"15min":1.667},"min":{"1min":0.42899999999999999,"5min":0.35399999999999998,"15min":0.34599999999999997},"max":{"1min":5.9219999999999997,"5min":10.051,"15min":10.051},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.079,"15min":1.5529999999999999},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":5.6470000000000002,"5min":10.119999999999999,"15min":10.119999999999999},"last":0.73499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.2929999999999999,"15min":1.661},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.8869999999999996,"5min":10.856999999999999,"15min":10.856999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.407,"15min":2.117},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.316},"max":{"1min":6.0389999999999997,"5min":11.75,"15min":12.295999999999999},"last":1.036}]},{"osd":6,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":1.351,"15min":1.748},"min":{"1min":0.438,"5min":0.371,"15min":0.371},"max":{"1min":5.7249999999999996,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.3140000000000001,"15min":1.4790000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":5.9130000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.5289999999999999,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.8410000000000002,"5min":10.007999999999999,"15min":14.161},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.3500000000000001,"15min":1.6950000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.41199999999999998},"max":{"1min":5.8029999999999999,"5min":9.8550000000000004,"15min":10.996},"last":0.875}]}]},{"osd":3,"up_from":103,"seq":442381631864,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4067928,"kb_used_data":3193360,"kb_used_omap":5,"kb_used_meta":874554,"kb_avail":89677224,"statfs":{"total":95995035648,"available":91829477376,"internally_reserved":0,"allocated":3270000640,"data_stored":3268827686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":895543470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":4.5439999999999996,"15min":2.98},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19500000000000001},"max":{"1min":6.6509999999999998,"5min":144.625,"15min":144.625},"last":4.0060000000000002},{"interface":"front","average":{"1min":2.0619999999999998,"5min":4.6230000000000002,"15min":2.9980000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.20000000000000001},"max":{"1min":9.0579999999999998,"5min":144.24100000000001,"15min":144.24100000000001},"last":15.657999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":4.0810000000000004,"15min":2.5070000000000001},"min":{"1min":0.23599999999999999,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":5.1550000000000002,"5min":143.34100000000001,"15min":143.34100000000001},"last":0.746},{"interface":"front","average":{"1min":2.0449999999999999,"5min":4.3810000000000002,"15min":2.7599999999999998},"min":{"1min":0.29599999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":6.6909999999999998,"5min":149.15700000000001,"15min":149.15700000000001},"last":3.3940000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.3149999999999999,"15min":1.4830000000000001},"min":{"1min":0.433,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":5.7069999999999999,"5min":10.148,"15min":10.148},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.2909999999999999,"15min":1.6020000000000001},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.147,"5min":13.069000000000001,"15min":13.069000000000001},"last":15.619999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.3169999999999999,"15min":1.365},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":5.5300000000000002,"5min":5.5300000000000002,"15min":8.8420000000000005},"last":3.3540000000000001},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.2549999999999999,"15min":1.4059999999999999},"min":{"1min":0.39200000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":5.359,"5min":8.8840000000000003,"15min":9.4359999999999999},"last":15.522}]},{"osd":5,"last update":"Mon Oct 9 04:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":2.0190000000000001,"15min":1.8069999999999999},"min":{"1min":0.55100000000000005,"5min":0.34799999999999998,"15min":0.33700000000000002},"max":{"1min":8.8450000000000006,"5min":10.372,"15min":10.372},"last":12.516},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.823,"15min":1.575},"min":{"1min":0.379,"5min":0.379,"15min":0.34200000000000003},"max":{"1min":8.9190000000000005,"5min":8.9190000000000005,"15min":9.4800000000000004},"last":15.573}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.7010000000000001,"15min":1.4990000000000001},"min":{"1min":0.40600000000000003,"5min":0.36799999999999999,"15min":0.32100000000000001},"max":{"1min":5.6749999999999998,"5min":10.102,"15min":10.102},"last":1.119},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.335,"15min":1.3620000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35399999999999998},"max":{"1min":5.4939999999999998,"5min":5.4939999999999998,"15min":8.8699999999999992},"last":3.3220000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.7450000000000001,"15min":1.6759999999999999},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.33100000000000002},"max":{"1min":5.4630000000000001,"5min":15.170999999999999,"15min":15.170999999999999},"last":3.25},{"interface":"front","average":{"1min":1.694,"5min":1.554,"15min":1.5760000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.36699999999999999},"max":{"1min":5.407,"5min":6.3019999999999996,"15min":9.2370000000000001},"last":3.2709999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019466,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":150,"seq":644245094648,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3363148,"kb_used_data":2407300,"kb_used_omap":6,"kb_used_meta":955833,"kb_avail":90382004,"statfs":{"total":95995035648,"available":92551172096,"internally_reserved":0,"allocated":2465075200,"data_stored":2464490411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":978773267},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":17.763999999999999,"5min":7.6349999999999998,"15min":3.7589999999999999},"min":{"1min":0.38400000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":363.64499999999998,"5min":363.64499999999998,"15min":363.64499999999998},"last":0.27800000000000002},{"interface":"front","average":{"1min":17.446999999999999,"5min":8.0510000000000002,"15min":4.0620000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":363.26900000000001,"5min":363.26900000000001,"15min":363.26900000000001},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":5.2009999999999996,"15min":2.903},"min":{"1min":0.308,"5min":0.308,"15min":0.28399999999999997},"max":{"1min":3.9820000000000002,"5min":130.762,"15min":130.762},"last":4.6050000000000004},{"interface":"front","average":{"1min":1.044,"5min":5.6680000000000001,"15min":3.2480000000000002},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.26400000000000001},"max":{"1min":4.5279999999999996,"5min":130.86199999999999,"15min":130.86199999999999},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":10.971,"5min":3.0600000000000001,"15min":2.0430000000000001},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":208.40100000000001,"5min":208.40100000000001,"15min":208.40100000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":11.106,"5min":3.3959999999999999,"15min":2.4660000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":208.078,"5min":208.078,"15min":208.078},"last":4.7149999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":3.2080000000000002,"15min":2.3439999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.5720000000000001,"5min":130.69200000000001,"15min":130.69200000000001},"last":4.6849999999999996},{"interface":"front","average":{"1min":1.286,"5min":3.2879999999999998,"15min":2.5179999999999998},"min":{"1min":0.443,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.5750000000000002,"5min":130.53399999999999,"15min":130.53399999999999},"last":0.66700000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.345,"15min":1.417},"min":{"1min":0.40200000000000002,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":1.702,"5min":12.734,"15min":12.734},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.115,"15min":1.262},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":1.484,"5min":8.3409999999999993,"15min":10.709},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.036,"15min":1.296},"min":{"1min":0.45200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.5820000000000001,"5min":7.2489999999999997,"15min":10.77},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.595,"15min":1.4990000000000001},"min":{"1min":0.41299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4409999999999998,"5min":14.223000000000001,"15min":14.223000000000001},"last":4.6470000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":2.1579999999999999,"15min":1.7909999999999999},"min":{"1min":0.46600000000000003,"5min":0.42499999999999999,"15min":0.32200000000000001},"max":{"1min":2.8250000000000002,"5min":12.706,"15min":12.706},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.012,"5min":1.7050000000000001,"15min":1.651},"min":{"1min":0.39600000000000002,"5min":0.39200000000000002,"15min":0.35699999999999998},"max":{"1min":2.3639999999999999,"5min":21.797000000000001,"15min":21.797000000000001},"last":0.71199999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4182441984,"data_stored":4182441984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2450784256,"data_stored":2450784256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2514681856,"data_stored":2514681856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3268476928,"data_stored":3268476928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2464350208,"data_stored":2464350208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2848325632,"data_stored":2848325632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2081161216,"data_stored":2081161216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:55:15.872 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:55:15.872 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:15.872 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-10-09T04:55:15.872 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:55:15.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:15.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:15.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:16.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:16.108 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:16.215 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:16.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:16.337 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:16.350 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:16.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total time run: 90.0101 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Total writes made: 105446 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Write size: 65536 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Object size: 65536 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Bandwidth (MB/sec): 73.2182 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Bandwidth: 28.2436 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max bandwidth (MB/sec): 172.812 2023-10-09T04:55:16.409 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min bandwidth (MB/sec): 41.5 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average IOPS: 1171 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev IOPS: 451.898 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max IOPS: 2765 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min IOPS: 664 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Average Latency(s): 0.0136533 2023-10-09T04:55:16.410 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Stddev Latency(s): 0.01423 2023-10-09T04:55:16.411 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Max latency(s): 0.508003 2023-10-09T04:55:16.411 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Min latency(s): 0.00123875 2023-10-09T04:55:16.416 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:55:16.478 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:16.508 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:16.614 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:16.643 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:16.750 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:16.781 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:16.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:16.927 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:16.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:17.032 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:17.060 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:17.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:17.193 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:17.299 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:17.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:17.353 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:17.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:17.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:17.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:17.569 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:17.592 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:17.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:17.724 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:17.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:17.868 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:17.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:17.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:17.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:18.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:18.124 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:18.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:18.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:17 smithi082 bash[77063]: cluster 2023-10-09T04:55:16.869139+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 792 op/s 2023-10-09T04:55:18.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:17 smithi082 bash[79733]: cluster 2023-10-09T04:55:16.869139+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 792 op/s 2023-10-09T04:55:18.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.240 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:17 smithi153 bash[59565]: cluster 2023-10-09T04:55:16.869139+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 49 MiB/s wr, 792 op/s 2023-10-09T04:55:18.256 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:18.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:18.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.506 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:18.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:18.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:18.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:18.746 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:18.770 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:18.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:18.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:18.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:19.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.163 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.267 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:19.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:19.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:19.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:19.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:19.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:19.743 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:55:19.743 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "osd_op(client.45806.0:108129 10.6 10:6a6d8192:::benchmark_data_smithi082_223129_object2680:head [delete] snapc 0=[] ondisk+write+known_if_redirected e307)", 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:55:19.741607+0000", 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 0.000662775, 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00069432200000000004, 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "client_info": { 2023-10-09T04:55:19.744 INFO:teuthology.orchestra.run.smithi153.stdout: "client": "client.45806", 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "tid": 108129 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741607+0000", 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T04:55:19.745 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741607+0000", 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741605+0000", 2023-10-09T04:55:19.746 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741614+0000", 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6201999999999998e-05 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741630+0000", 2023-10-09T04:55:19.747 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.5340000000000002e-06 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741638+0000", 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9675e-05 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741697+0000", 2023-10-09T04:55:19.748 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00016460699999999999 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.741862+0000", 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00015047500000000001 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:55:19.742012+0000", 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00040495899999999998 2023-10-09T04:55:19.749 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout: ], 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 1 2023-10-09T04:55:19.750 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:55:19.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:19.801 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:19] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.33.4" 2023-10-09T04:55:19.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:19.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:19.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:19.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:20.062 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:20.087 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:20.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:19 smithi153 bash[59565]: cluster 2023-10-09T04:55:18.870758+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 95 B/s rd, 51 MiB/s wr, 910 op/s 2023-10-09T04:55:20.187 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:19 smithi082 bash[77063]: cluster 2023-10-09T04:55:18.870758+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 95 B/s rd, 51 MiB/s wr, 910 op/s 2023-10-09T04:55:20.187 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:19 smithi082 bash[79733]: cluster 2023-10-09T04:55:18.870758+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 95 B/s rd, 51 MiB/s wr, 910 op/s 2023-10-09T04:55:20.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:20.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:20.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:20.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:20.453 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:20.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:20.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:20.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:20.696 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:55:20.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:20.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:20.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:20.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:20.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:21.019 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:21.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:21.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:21.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:21.310 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:21.419 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:21.444 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:21.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:21.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.583 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:21.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.718 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:21.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:21.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:21.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:21.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:21.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:21.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:22.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:22.124 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:22.148 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:22 smithi082 bash[77063]: cluster 2023-10-09T04:55:20.871207+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 698 op/s 2023-10-09T04:55:22.148 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:22 smithi082 bash[79733]: cluster 2023-10-09T04:55:20.871207+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 698 op/s 2023-10-09T04:55:22.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:22.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:22.364 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.390 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:22.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:22 smithi153 bash[59565]: cluster 2023-10-09T04:55:20.871207+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 698 op/s 2023-10-09T04:55:22.493 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:22.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:22.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:22.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:22.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:22.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:22.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:22.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:23.023 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:23.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:23.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:23.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:23.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:23.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:23.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:23.601 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:23.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.734 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:23.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:23.875 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:23.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:23.979 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:24.016 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:24.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:24.253 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.273 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:24.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:24.387 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:24 smithi153 bash[59565]: cluster 2023-10-09T04:55:22.871807+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 806 op/s 2023-10-09T04:55:24.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:24 smithi082 bash[77063]: cluster 2023-10-09T04:55:22.871807+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 806 op/s 2023-10-09T04:55:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:24 smithi082 bash[79733]: cluster 2023-10-09T04:55:22.871807+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 34 MiB/s wr, 806 op/s 2023-10-09T04:55:24.514 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.541 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:24.645 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.669 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:24.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:24.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:24.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:24.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:24.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:24.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:25.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:25.092 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:25.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:25.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:25.322 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:55:25.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:25.352 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:25.370 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:55:25.419 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:55:25.455 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:25.471 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:55:25.490 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:25.502 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:55:25.513 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:25.545 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:55:25.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:25.597 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:55:25.628 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:25.634 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:55:25.645 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.645 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.645 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.646 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.646 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.646 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.646 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.646 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:55:25.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:25.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:25.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:25.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:25.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:25.995 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:26.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:26.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:26.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:26.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:26.177 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:55:26.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:26.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:26.406 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:26.415 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:26 smithi153 bash[59565]: cluster 2023-10-09T04:55:24.872358+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 810 op/s 2023-10-09T04:55:26.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:26 smithi153 bash[59565]: audit 2023-10-09T04:55:25.275697+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.82:0/3553093218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:26.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:26 smithi153 bash[59565]: audit 2023-10-09T04:55:25.979572+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.82:0/3489170988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:26 smithi153 bash[59565]: audit 2023-10-09T04:55:25.980305+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.431 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[77063]: cluster 2023-10-09T04:55:24.872358+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 810 op/s 2023-10-09T04:55:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[77063]: audit 2023-10-09T04:55:25.275697+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.82:0/3553093218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[77063]: audit 2023-10-09T04:55:25.979572+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.82:0/3489170988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[77063]: audit 2023-10-09T04:55:25.980305+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[79733]: cluster 2023-10-09T04:55:24.872358+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 810 op/s 2023-10-09T04:55:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[79733]: audit 2023-10-09T04:55:25.275697+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.82:0/3553093218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[79733]: audit 2023-10-09T04:55:25.979572+0000 mon.b (mon.1) 33 : audit 1 from='client.? 172.21.15.82:0/3489170988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:26 smithi082 bash[79733]: audit 2023-10-09T04:55:25.980305+0000 mon.a (mon.0) 1809 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:26.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:26.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:26.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:26.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:26.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:26.825 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:26.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:26.930 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:26.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:27.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:27.176 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:27.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:27.313 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:27.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:27.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:27 smithi153 bash[59565]: cluster 2023-10-09T04:55:26.150528+0000 mon.a (mon.0) 1810 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:27 smithi153 bash[59565]: cluster 2023-10-09T04:55:26.165349+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:27 smithi153 bash[59565]: cluster 2023-10-09T04:55:26.165794+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:27 smithi153 bash[59565]: audit 2023-10-09T04:55:26.169200+0000 mon.a (mon.0) 1811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:27 smithi153 bash[59565]: cluster 2023-10-09T04:55:26.169264+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:55:27.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[77063]: cluster 2023-10-09T04:55:26.150528+0000 mon.a (mon.0) 1810 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[77063]: cluster 2023-10-09T04:55:26.165349+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[77063]: cluster 2023-10-09T04:55:26.165794+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[77063]: audit 2023-10-09T04:55:26.169200+0000 mon.a (mon.0) 1811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[77063]: cluster 2023-10-09T04:55:26.169264+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[79733]: cluster 2023-10-09T04:55:26.150528+0000 mon.a (mon.0) 1810 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[79733]: cluster 2023-10-09T04:55:26.165349+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[79733]: cluster 2023-10-09T04:55:26.165794+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[79733]: audit 2023-10-09T04:55:26.169200+0000 mon.a (mon.0) 1811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:27.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:27 smithi082 bash[79733]: cluster 2023-10-09T04:55:26.169264+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:55:27.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:27.556 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:27.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:27.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:27.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:27.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:27.842 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:27.945 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:27.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:27.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:27.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:28.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:28.097 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:28.203 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:28.240 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:28.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.373 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:28.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:28 smithi082 bash[77063]: cluster 2023-10-09T04:55:26.872891+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 805 op/s 2023-10-09T04:55:28.462 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:28 smithi082 bash[79733]: cluster 2023-10-09T04:55:26.872891+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 805 op/s 2023-10-09T04:55:28.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:28.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:28.623 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:28 smithi153 bash[59565]: cluster 2023-10-09T04:55:26.872891+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 805 op/s 2023-10-09T04:55:28.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.784 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:28.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:28.922 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:28.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:29.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:29.052 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:29.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:29.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:29.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:29.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:29.421 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:29.449 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:29.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:29.552 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:29.575 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:29.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:29.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:29.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:29.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:29.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:29.948 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:29.958 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:29] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.33.4" 2023-10-09T04:55:29.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:29.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:30.082 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:30.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:30.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[77063]: cluster 2023-10-09T04:55:28.873513+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 813 KiB/s wr, 790 op/s 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[77063]: audit 2023-10-09T04:55:29.303745+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[77063]: audit 2023-10-09T04:55:29.307547+0000 mon.a (mon.0) 1814 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[79733]: cluster 2023-10-09T04:55:28.873513+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 813 KiB/s wr, 790 op/s 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[79733]: audit 2023-10-09T04:55:29.303745+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:30.225 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:30 smithi082 bash[79733]: audit 2023-10-09T04:55:29.307547+0000 mon.a (mon.0) 1814 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.250 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:30.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:30.299 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:55:30.299 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:55:30.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:30.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:30.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:30 smithi153 bash[59565]: cluster 2023-10-09T04:55:28.873513+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 813 KiB/s wr, 790 op/s 2023-10-09T04:55:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:30 smithi153 bash[59565]: audit 2023-10-09T04:55:29.303745+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:30 smithi153 bash[59565]: audit 2023-10-09T04:55:29.307547+0000 mon.a (mon.0) 1814 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:30.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:55:30.488 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:30.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:55:30.516 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:30.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:30.590 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:55:30.593 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:55:30.621 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:30.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:30.751 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:30.786 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:30.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:30.919 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:30.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:31.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.057 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.165 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:31.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:31.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:31.342 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:31.440 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:55:31.445 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.511 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.614 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:31.760 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.794 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1367,"stamp":"2023-10-09T04:55:28.873086+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6334322635,"num_objects":96610,"num_object_clones":0,"num_object_copies":289830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96610,"num_whiteouts":0,"num_read":6288,"num_read_kb":12724,"num_write":115418,"num_write_kb":6763704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":8398438,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115182,"ondisk_log_size":115182,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":24143108,"kb_used_data":18756812,"kb_used_omap":47,"kb_used_meta":5385936,"kb_avail":632072956,"statfs":{"total":671965249536,"available":647242706944,"internally_reserved":0,"allocated":19206975488,"data_stored":19200505045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":5515198877},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,5,0,0,6,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-501481472,"num_objects":-7652,"num_object_clones":0,"num_object_copies":-22956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7906,"num_write_kb":8128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002750"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":41,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.200574+0000","last_change":"2023-10-09T04:51:14.108186+0000","last_active":"2023-10-09T04:55:08.200574+0000","last_peered":"2023-10-09T04:55:08.200574+0000","last_clean":"2023-10-09T04:55:08.200574+0000","last_became_active":"2023-10-09T04:51:14.107642+0000","last_became_peered":"2023-10-09T04:51:14.107642+0000","last_unstale":"2023-10-09T04:55:08.200574+0000","last_undegraded":"2023-10-09T04:55:08.200574+0000","last_fullsized":"2023-10-09T04:55:08.200574+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:32:39.703389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":275,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201300+0000","last_change":"2023-10-09T04:51:14.111673+0000","last_active":"2023-10-09T04:55:08.201300+0000","last_peered":"2023-10-09T04:55:08.201300+0000","last_clean":"2023-10-09T04:55:08.201300+0000","last_became_active":"2023-10-09T04:51:14.111303+0000","last_became_peered":"2023-10-09T04:51:14.111303+0000","last_unstale":"2023-10-09T04:55:08.201300+0000","last_undegraded":"2023-10-09T04:55:08.201300+0000","last_fullsized":"2023-10-09T04:55:08.201300+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:38.607841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":271,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229445+0000","last_change":"2023-10-09T04:51:14.150395+0000","last_active":"2023-10-09T04:54:47.229445+0000","last_peered":"2023-10-09T04:54:47.229445+0000","last_clean":"2023-10-09T04:54:47.229445+0000","last_became_active":"2023-10-09T04:51:14.149329+0000","last_became_peered":"2023-10-09T04:51:14.149329+0000","last_unstale":"2023-10-09T04:54:47.229445+0000","last_undegraded":"2023-10-09T04:54:47.229445+0000","last_fullsized":"2023-10-09T04:54:47.229445+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:50.627469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":288,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.185973+0000","last_change":"2023-10-09T04:40:47.864154+0000","last_active":"2023-10-09T04:55:08.185973+0000","last_peered":"2023-10-09T04:55:08.185973+0000","last_clean":"2023-10-09T04:55:08.185973+0000","last_became_active":"2023-10-09T04:40:47.863863+0000","last_became_peered":"2023-10-09T04:40:47.863863+0000","last_unstale":"2023-10-09T04:55:08.185973+0000","last_undegraded":"2023-10-09T04:55:08.185973+0000","last_fullsized":"2023-10-09T04:55:08.185973+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:19.105559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":290,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.185739+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:55:08.185739+0000","last_peered":"2023-10-09T04:55:08.185739+0000","last_clean":"2023-10-09T04:55:08.185739+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:55:08.185739+0000","last_undegraded":"2023-10-09T04:55:08.185739+0000","last_fullsized":"2023-10-09T04:55:08.185739+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":263,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193301+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:55:08.193301+0000","last_peered":"2023-10-09T04:55:08.193301+0000","last_clean":"2023-10-09T04:55:08.193301+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:55:08.193301+0000","last_undegraded":"2023-10-09T04:55:08.193301+0000","last_fullsized":"2023-10-09T04:55:08.193301+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":282,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.186083+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:55:08.186083+0000","last_peered":"2023-10-09T04:55:08.186083+0000","last_clean":"2023-10-09T04:55:08.186083+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:55:08.186083+0000","last_undegraded":"2023-10-09T04:55:08.186083+0000","last_fullsized":"2023-10-09T04:55:08.186083+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":229,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.189189+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:55:08.189189+0000","last_peered":"2023-10-09T04:55:08.189189+0000","last_clean":"2023-10-09T04:55:08.189189+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:55:08.189189+0000","last_undegraded":"2023-10-09T04:55:08.189189+0000","last_fullsized":"2023-10-09T04:55:08.189189+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232377+0000","last_change":"2023-10-09T04:40:47.819489+0000","last_active":"2023-10-09T04:54:47.232377+0000","last_peered":"2023-10-09T04:54:47.232377+0000","last_clean":"2023-10-09T04:54:47.232377+0000","last_became_active":"2023-10-09T04:40:47.819137+0000","last_became_peered":"2023-10-09T04:40:47.819137+0000","last_unstale":"2023-10-09T04:54:47.232377+0000","last_undegraded":"2023-10-09T04:54:47.232377+0000","last_fullsized":"2023-10-09T04:54:47.232377+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":123,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201183+0000","last_change":"2023-10-09T04:40:47.823764+0000","last_active":"2023-10-09T04:55:08.201183+0000","last_peered":"2023-10-09T04:55:08.201183+0000","last_clean":"2023-10-09T04:55:08.201183+0000","last_became_active":"2023-10-09T04:40:47.821988+0000","last_became_peered":"2023-10-09T04:40:47.821988+0000","last_unstale":"2023-10-09T04:55:08.201183+0000","last_undegraded":"2023-10-09T04:55:08.201183+0000","last_fullsized":"2023-10-09T04:55:08.201183+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":144,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201436+0000","last_change":"2023-10-09T04:51:14.339838+0000","last_active":"2023-10-09T04:55:08.201436+0000","last_peered":"2023-10-09T04:55:08.201436+0000","last_clean":"2023-10-09T04:55:08.201436+0000","last_became_active":"2023-10-09T04:51:14.150185+0000","last_became_peered":"2023-10-09T04:51:14.150185+0000","last_unstale":"2023-10-09T04:55:08.201436+0000","last_undegraded":"2023-10-09T04:55:08.201436+0000","last_fullsized":"2023-10-09T04:55:08.201436+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":251,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193274+0000","last_change":"2023-10-09T04:50:44.052710+0000","last_active":"2023-10-09T04:55:08.193274+0000","last_peered":"2023-10-09T04:55:08.193274+0000","last_clean":"2023-10-09T04:55:08.193274+0000","last_became_active":"2023-10-09T04:50:44.052312+0000","last_became_peered":"2023-10-09T04:50:44.052312+0000","last_unstale":"2023-10-09T04:55:08.193274+0000","last_undegraded":"2023-10-09T04:55:08.193274+0000","last_fullsized":"2023-10-09T04:55:08.193274+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":251,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201545+0000","last_change":"2023-10-09T04:51:14.162470+0000","last_active":"2023-10-09T04:55:08.201545+0000","last_peered":"2023-10-09T04:55:08.201545+0000","last_clean":"2023-10-09T04:55:08.201545+0000","last_became_active":"2023-10-09T04:51:14.129144+0000","last_became_peered":"2023-10-09T04:51:14.129144+0000","last_unstale":"2023-10-09T04:55:08.201545+0000","last_undegraded":"2023-10-09T04:55:08.201545+0000","last_fullsized":"2023-10-09T04:55:08.201545+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191664+0000","last_change":"2023-10-09T04:51:14.113752+0000","last_active":"2023-10-09T04:55:08.191664+0000","last_peered":"2023-10-09T04:55:08.191664+0000","last_clean":"2023-10-09T04:55:08.191664+0000","last_became_active":"2023-10-09T04:51:14.113290+0000","last_became_peered":"2023-10-09T04:51:14.113290+0000","last_unstale":"2023-10-09T04:55:08.191664+0000","last_undegraded":"2023-10-09T04:55:08.191664+0000","last_fullsized":"2023-10-09T04:55:08.191664+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":159,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201406+0000","last_change":"2023-10-09T04:51:14.179666+0000","last_active":"2023-10-09T04:55:08.201406+0000","last_peered":"2023-10-09T04:55:08.201406+0000","last_clean":"2023-10-09T04:55:08.201406+0000","last_became_active":"2023-10-09T04:51:14.130965+0000","last_became_peered":"2023-10-09T04:51:14.130965+0000","last_unstale":"2023-10-09T04:55:08.201406+0000","last_undegraded":"2023-10-09T04:55:08.201406+0000","last_fullsized":"2023-10-09T04:55:08.201406+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":210,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201653+0000","last_change":"2023-10-09T04:50:44.064722+0000","last_active":"2023-10-09T04:55:08.201653+0000","last_peered":"2023-10-09T04:55:08.201653+0000","last_clean":"2023-10-09T04:55:08.201653+0000","last_became_active":"2023-10-09T04:50:44.064373+0000","last_became_peered":"2023-10-09T04:50:44.064373+0000","last_unstale":"2023-10-09T04:55:08.201653+0000","last_undegraded":"2023-10-09T04:55:08.201653+0000","last_fullsized":"2023-10-09T04:55:08.201653+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":286,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191080+0000","last_change":"2023-10-09T04:40:47.823670+0000","last_active":"2023-10-09T04:55:08.191080+0000","last_peered":"2023-10-09T04:55:08.191080+0000","last_clean":"2023-10-09T04:55:08.191080+0000","last_became_active":"2023-10-09T04:40:47.814169+0000","last_became_peered":"2023-10-09T04:40:47.814169+0000","last_unstale":"2023-10-09T04:55:08.191080+0000","last_undegraded":"2023-10-09T04:55:08.191080+0000","last_fullsized":"2023-10-09T04:55:08.191080+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:32.876684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232357+0000","last_change":"2023-10-09T04:53:31.658083+0000","last_active":"2023-10-09T04:54:47.232357+0000","last_peered":"2023-10-09T04:54:47.232357+0000","last_clean":"2023-10-09T04:54:47.232357+0000","last_became_active":"2023-10-09T04:53:31.657030+0000","last_became_peered":"2023-10-09T04:53:31.657030+0000","last_unstale":"2023-10-09T04:54:47.232357+0000","last_undegraded":"2023-10-09T04:54:47.232357+0000","last_fullsized":"2023-10-09T04:54:47.232357+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":132,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201277+0000","last_change":"2023-10-09T04:51:14.129515+0000","last_active":"2023-10-09T04:55:08.201277+0000","last_peered":"2023-10-09T04:55:08.201277+0000","last_clean":"2023-10-09T04:55:08.201277+0000","last_became_active":"2023-10-09T04:51:14.128945+0000","last_became_peered":"2023-10-09T04:51:14.128945+0000","last_unstale":"2023-10-09T04:55:08.201277+0000","last_undegraded":"2023-10-09T04:55:08.201277+0000","last_fullsized":"2023-10-09T04:55:08.201277+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":285,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.186055+0000","last_change":"2023-10-09T04:40:47.864771+0000","last_active":"2023-10-09T04:55:08.186055+0000","last_peered":"2023-10-09T04:55:08.186055+0000","last_clean":"2023-10-09T04:55:08.186055+0000","last_became_active":"2023-10-09T04:40:47.864483+0000","last_became_peered":"2023-10-09T04:40:47.864483+0000","last_unstale":"2023-10-09T04:55:08.186055+0000","last_undegraded":"2023-10-09T04:55:08.186055+0000","last_fullsized":"2023-10-09T04:55:08.186055+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:43.144539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":272,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191398+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:55:08.191398+0000","last_peered":"2023-10-09T04:55:08.191398+0000","last_clean":"2023-10-09T04:55:08.191398+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:55:08.191398+0000","last_undegraded":"2023-10-09T04:55:08.191398+0000","last_fullsized":"2023-10-09T04:55:08.191398+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231873+0000","last_change":"2023-10-09T04:41:21.460517+0000","last_active":"2023-10-09T04:54:47.231873+0000","last_peered":"2023-10-09T04:54:47.231873+0000","last_clean":"2023-10-09T04:54:47.231873+0000","last_became_active":"2023-10-09T04:41:21.460190+0000","last_became_peered":"2023-10-09T04:41:21.460190+0000","last_unstale":"2023-10-09T04:54:47.231873+0000","last_undegraded":"2023-10-09T04:54:47.231873+0000","last_fullsized":"2023-10-09T04:54:47.231873+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.412075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"307'7013","reported_seq":7035,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:24.985446+0000","last_change":"2023-10-09T04:53:24.353425+0000","last_active":"2023-10-09T04:55:24.985446+0000","last_peered":"2023-10-09T04:55:24.985446+0000","last_clean":"2023-10-09T04:55:24.985446+0000","last_became_active":"2023-10-09T04:53:24.352877+0000","last_became_peered":"2023-10-09T04:53:24.352877+0000","last_unstale":"2023-10-09T04:55:24.985446+0000","last_undegraded":"2023-10-09T04:55:24.985446+0000","last_fullsized":"2023-10-09T04:55:24.985446+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404815872,"num_objects":6177,"num_object_clones":0,"num_object_copies":18531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7013,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":140,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.202102+0000","last_change":"2023-10-09T04:51:14.384573+0000","last_active":"2023-10-09T04:55:08.202102+0000","last_peered":"2023-10-09T04:55:08.202102+0000","last_clean":"2023-10-09T04:55:08.202102+0000","last_became_active":"2023-10-09T04:51:14.155381+0000","last_became_peered":"2023-10-09T04:51:14.155381+0000","last_unstale":"2023-10-09T04:55:08.202102+0000","last_undegraded":"2023-10-09T04:55:08.202102+0000","last_fullsized":"2023-10-09T04:55:08.202102+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231828+0000","last_change":"2023-10-09T04:40:47.820403+0000","last_active":"2023-10-09T04:54:47.231828+0000","last_peered":"2023-10-09T04:54:47.231828+0000","last_clean":"2023-10-09T04:54:47.231828+0000","last_became_active":"2023-10-09T04:40:47.819966+0000","last_became_peered":"2023-10-09T04:40:47.819966+0000","last_unstale":"2023-10-09T04:54:47.231828+0000","last_undegraded":"2023-10-09T04:54:47.231828+0000","last_fullsized":"2023-10-09T04:54:47.231828+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"308'7278","reported_seq":7300,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.663763+0000","last_change":"2023-10-09T04:53:24.139931+0000","last_active":"2023-10-09T04:55:28.663763+0000","last_peered":"2023-10-09T04:55:28.663763+0000","last_clean":"2023-10-09T04:55:28.663763+0000","last_became_active":"2023-10-09T04:53:24.139560+0000","last_became_peered":"2023-10-09T04:53:24.139560+0000","last_unstale":"2023-10-09T04:55:28.663763+0000","last_undegraded":"2023-10-09T04:55:28.663763+0000","last_fullsized":"2023-10-09T04:55:28.663763+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7278,"log_dups_size":0,"ondisk_log_size":7278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7278,"num_write_kb":424448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":153,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201876+0000","last_change":"2023-10-09T04:51:14.431351+0000","last_active":"2023-10-09T04:55:08.201876+0000","last_peered":"2023-10-09T04:55:08.201876+0000","last_clean":"2023-10-09T04:55:08.201876+0000","last_became_active":"2023-10-09T04:51:14.156447+0000","last_became_peered":"2023-10-09T04:51:14.156447+0000","last_unstale":"2023-10-09T04:55:08.201876+0000","last_undegraded":"2023-10-09T04:55:08.201876+0000","last_fullsized":"2023-10-09T04:55:08.201876+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":113,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229309+0000","last_change":"2023-10-09T04:51:14.150212+0000","last_active":"2023-10-09T04:54:47.229309+0000","last_peered":"2023-10-09T04:54:47.229309+0000","last_clean":"2023-10-09T04:54:47.229309+0000","last_became_active":"2023-10-09T04:51:14.149425+0000","last_became_peered":"2023-10-09T04:51:14.149425+0000","last_unstale":"2023-10-09T04:54:47.229309+0000","last_undegraded":"2023-10-09T04:54:47.229309+0000","last_fullsized":"2023-10-09T04:54:47.229309+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:56.783749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"308'7158","reported_seq":7182,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:27.157968+0000","last_change":"2023-10-09T04:53:24.349405+0000","last_active":"2023-10-09T04:55:27.157968+0000","last_peered":"2023-10-09T04:55:27.157968+0000","last_clean":"2023-10-09T04:55:27.157968+0000","last_became_active":"2023-10-09T04:53:24.348702+0000","last_became_peered":"2023-10-09T04:53:24.348702+0000","last_unstale":"2023-10-09T04:55:27.157968+0000","last_undegraded":"2023-10-09T04:55:27.157968+0000","last_fullsized":"2023-10-09T04:55:27.157968+0000","mapping_epoch":293,"log_start":"301'100","ondisk_log_start":"301'100","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:31.168505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397934592,"num_objects":6072,"num_object_clones":0,"num_object_copies":18216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7158,"num_write_kb":423360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201546+0000","last_change":"2023-10-09T04:40:48.125689+0000","last_active":"2023-10-09T04:55:08.201546+0000","last_peered":"2023-10-09T04:55:08.201546+0000","last_clean":"2023-10-09T04:55:08.201546+0000","last_became_active":"2023-10-09T04:40:47.822509+0000","last_became_peered":"2023-10-09T04:40:47.822509+0000","last_unstale":"2023-10-09T04:55:08.201546+0000","last_undegraded":"2023-10-09T04:55:08.201546+0000","last_fullsized":"2023-10-09T04:55:08.201546+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":292,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191821+0000","last_change":"2023-10-09T04:40:47.825700+0000","last_active":"2023-10-09T04:55:08.191821+0000","last_peered":"2023-10-09T04:55:08.191821+0000","last_clean":"2023-10-09T04:55:08.191821+0000","last_became_active":"2023-10-09T04:40:47.824167+0000","last_became_peered":"2023-10-09T04:40:47.824167+0000","last_unstale":"2023-10-09T04:55:08.191821+0000","last_undegraded":"2023-10-09T04:55:08.191821+0000","last_fullsized":"2023-10-09T04:55:08.191821+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"308'7189","reported_seq":7213,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:27.570476+0000","last_change":"2023-10-09T04:53:24.127013+0000","last_active":"2023-10-09T04:55:27.570476+0000","last_peered":"2023-10-09T04:55:27.570476+0000","last_clean":"2023-10-09T04:55:27.570476+0000","last_became_active":"2023-10-09T04:53:24.126651+0000","last_became_peered":"2023-10-09T04:53:24.126651+0000","last_unstale":"2023-10-09T04:55:27.570476+0000","last_undegraded":"2023-10-09T04:55:27.570476+0000","last_fullsized":"2023-10-09T04:55:27.570476+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7189,"log_dups_size":0,"ondisk_log_size":7189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:33.048421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399835136,"num_objects":6101,"num_object_clones":0,"num_object_copies":18303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7189,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":240,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231953+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:54:47.231953+0000","last_peered":"2023-10-09T04:54:47.231953+0000","last_clean":"2023-10-09T04:54:47.231953+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:54:47.231953+0000","last_undegraded":"2023-10-09T04:54:47.231953+0000","last_fullsized":"2023-10-09T04:54:47.231953+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":305,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.189634+0000","last_change":"2023-10-09T04:51:14.146161+0000","last_active":"2023-10-09T04:55:08.189634+0000","last_peered":"2023-10-09T04:55:08.189634+0000","last_clean":"2023-10-09T04:55:08.189634+0000","last_became_active":"2023-10-09T04:51:14.145388+0000","last_became_peered":"2023-10-09T04:51:14.145388+0000","last_unstale":"2023-10-09T04:55:08.189634+0000","last_undegraded":"2023-10-09T04:55:08.189634+0000","last_fullsized":"2023-10-09T04:55:08.189634+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:03.104400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"308'7116","reported_seq":7140,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:27.147370+0000","last_change":"2023-10-09T04:53:24.350004+0000","last_active":"2023-10-09T04:55:27.147370+0000","last_peered":"2023-10-09T04:55:27.147370+0000","last_clean":"2023-10-09T04:55:27.147370+0000","last_became_active":"2023-10-09T04:53:24.349539+0000","last_became_peered":"2023-10-09T04:53:24.349539+0000","last_unstale":"2023-10-09T04:55:27.147370+0000","last_undegraded":"2023-10-09T04:55:27.147370+0000","last_fullsized":"2023-10-09T04:55:27.147370+0000","mapping_epoch":293,"log_start":"301'100","ondisk_log_start":"301'100","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:55.535031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396099584,"num_objects":6044,"num_object_clones":0,"num_object_copies":18132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7116,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":244,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.189728+0000","last_change":"2023-10-09T04:51:14.181439+0000","last_active":"2023-10-09T04:55:08.189728+0000","last_peered":"2023-10-09T04:55:08.189728+0000","last_clean":"2023-10-09T04:55:08.189728+0000","last_became_active":"2023-10-09T04:51:14.150700+0000","last_became_peered":"2023-10-09T04:51:14.150700+0000","last_unstale":"2023-10-09T04:55:08.189728+0000","last_undegraded":"2023-10-09T04:55:08.189728+0000","last_fullsized":"2023-10-09T04:55:08.189728+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:26.754849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191566+0000","last_change":"2023-10-09T04:51:14.151318+0000","last_active":"2023-10-09T04:55:08.191566+0000","last_peered":"2023-10-09T04:55:08.191566+0000","last_clean":"2023-10-09T04:55:08.191566+0000","last_became_active":"2023-10-09T04:51:14.150870+0000","last_became_peered":"2023-10-09T04:51:14.150870+0000","last_unstale":"2023-10-09T04:55:08.191566+0000","last_undegraded":"2023-10-09T04:55:08.191566+0000","last_fullsized":"2023-10-09T04:55:08.191566+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"308'7176","reported_seq":7200,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:27.604839+0000","last_change":"2023-10-09T04:53:24.136245+0000","last_active":"2023-10-09T04:55:27.604839+0000","last_peered":"2023-10-09T04:55:27.604839+0000","last_clean":"2023-10-09T04:55:27.604839+0000","last_became_active":"2023-10-09T04:53:24.135900+0000","last_became_peered":"2023-10-09T04:53:24.135900+0000","last_unstale":"2023-10-09T04:55:27.604839+0000","last_undegraded":"2023-10-09T04:55:27.604839+0000","last_fullsized":"2023-10-09T04:55:27.604839+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7176,"log_dups_size":0,"ondisk_log_size":7176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:27.074775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397934592,"num_objects":6072,"num_object_clones":0,"num_object_copies":18216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7176,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":284,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193046+0000","last_change":"2023-10-09T04:51:14.149198+0000","last_active":"2023-10-09T04:55:08.193046+0000","last_peered":"2023-10-09T04:55:08.193046+0000","last_clean":"2023-10-09T04:55:08.193046+0000","last_became_active":"2023-10-09T04:51:14.148826+0000","last_became_peered":"2023-10-09T04:51:14.148826+0000","last_unstale":"2023-10-09T04:55:08.193046+0000","last_undegraded":"2023-10-09T04:55:08.193046+0000","last_fullsized":"2023-10-09T04:55:08.193046+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":191,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201247+0000","last_change":"2023-10-09T04:51:14.109636+0000","last_active":"2023-10-09T04:55:08.201247+0000","last_peered":"2023-10-09T04:55:08.201247+0000","last_clean":"2023-10-09T04:55:08.201247+0000","last_became_active":"2023-10-09T04:51:14.109258+0000","last_became_peered":"2023-10-09T04:51:14.109258+0000","last_unstale":"2023-10-09T04:55:08.201247+0000","last_undegraded":"2023-10-09T04:55:08.201247+0000","last_fullsized":"2023-10-09T04:55:08.201247+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.321575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"308'7200","reported_seq":7222,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.671434+0000","last_change":"2023-10-09T04:53:24.349881+0000","last_active":"2023-10-09T04:55:28.671434+0000","last_peered":"2023-10-09T04:55:28.671434+0000","last_clean":"2023-10-09T04:55:28.671434+0000","last_became_active":"2023-10-09T04:53:24.348953+0000","last_became_peered":"2023-10-09T04:53:24.348953+0000","last_unstale":"2023-10-09T04:55:28.671434+0000","last_undegraded":"2023-10-09T04:55:28.671434+0000","last_fullsized":"2023-10-09T04:55:28.671434+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388235264,"num_objects":5924,"num_object_clones":0,"num_object_copies":17772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7200,"num_write_kb":419968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":141,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201902+0000","last_change":"2023-10-09T04:51:14.125565+0000","last_active":"2023-10-09T04:55:08.201902+0000","last_peered":"2023-10-09T04:55:08.201902+0000","last_clean":"2023-10-09T04:55:08.201902+0000","last_became_active":"2023-10-09T04:51:14.125345+0000","last_became_peered":"2023-10-09T04:51:14.125345+0000","last_unstale":"2023-10-09T04:55:08.201902+0000","last_undegraded":"2023-10-09T04:55:08.201902+0000","last_fullsized":"2023-10-09T04:55:08.201902+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:14.172443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193000+0000","last_change":"2023-10-09T04:51:14.146838+0000","last_active":"2023-10-09T04:55:08.193000+0000","last_peered":"2023-10-09T04:55:08.193000+0000","last_clean":"2023-10-09T04:55:08.193000+0000","last_became_active":"2023-10-09T04:51:14.146457+0000","last_became_peered":"2023-10-09T04:51:14.146457+0000","last_unstale":"2023-10-09T04:55:08.193000+0000","last_undegraded":"2023-10-09T04:55:08.193000+0000","last_fullsized":"2023-10-09T04:55:08.193000+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"308'7173","reported_seq":7196,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.433494+0000","last_change":"2023-10-09T04:53:24.350320+0000","last_active":"2023-10-09T04:55:26.433494+0000","last_peered":"2023-10-09T04:55:26.433494+0000","last_clean":"2023-10-09T04:55:26.433494+0000","last_became_active":"2023-10-09T04:53:24.349443+0000","last_became_peered":"2023-10-09T04:53:24.349443+0000","last_unstale":"2023-10-09T04:55:26.433494+0000","last_undegraded":"2023-10-09T04:55:26.433494+0000","last_fullsized":"2023-10-09T04:55:26.433494+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7173,"log_dups_size":0,"ondisk_log_size":7173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7173,"num_write_kb":426688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":284,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.229393+0000","last_change":"2023-10-09T04:41:21.461484+0000","last_active":"2023-10-09T04:54:47.229393+0000","last_peered":"2023-10-09T04:54:47.229393+0000","last_clean":"2023-10-09T04:54:47.229393+0000","last_became_active":"2023-10-09T04:41:21.460784+0000","last_became_peered":"2023-10-09T04:41:21.460784+0000","last_unstale":"2023-10-09T04:54:47.229393+0000","last_undegraded":"2023-10-09T04:54:47.229393+0000","last_fullsized":"2023-10-09T04:54:47.229393+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:57:50.589473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":38,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201947+0000","last_change":"2023-10-09T04:51:14.155214+0000","last_active":"2023-10-09T04:55:08.201947+0000","last_peered":"2023-10-09T04:55:08.201947+0000","last_clean":"2023-10-09T04:55:08.201947+0000","last_became_active":"2023-10-09T04:51:14.154876+0000","last_became_peered":"2023-10-09T04:51:14.154876+0000","last_unstale":"2023-10-09T04:55:08.201947+0000","last_undegraded":"2023-10-09T04:55:08.201947+0000","last_fullsized":"2023-10-09T04:55:08.201947+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.c","version":"308'7037","reported_seq":7060,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.426076+0000","last_change":"2023-10-09T04:53:24.135648+0000","last_active":"2023-10-09T04:55:26.426076+0000","last_peered":"2023-10-09T04:55:26.426076+0000","last_clean":"2023-10-09T04:55:26.426076+0000","last_became_active":"2023-10-09T04:53:24.135052+0000","last_became_peered":"2023-10-09T04:53:24.135052+0000","last_unstale":"2023-10-09T04:55:26.426076+0000","last_undegraded":"2023-10-09T04:55:26.426076+0000","last_fullsized":"2023-10-09T04:55:26.426076+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398393344,"num_objects":6079,"num_object_clones":0,"num_object_copies":18237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7037,"num_write_kb":419712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":473,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.187474+0000","last_change":"2023-10-09T04:50:44.050055+0000","last_active":"2023-10-09T04:55:08.187474+0000","last_peered":"2023-10-09T04:55:08.187474+0000","last_clean":"2023-10-09T04:55:08.187474+0000","last_became_active":"2023-10-09T04:50:44.049620+0000","last_became_peered":"2023-10-09T04:50:44.049620+0000","last_unstale":"2023-10-09T04:55:08.187474+0000","last_undegraded":"2023-10-09T04:55:08.187474+0000","last_fullsized":"2023-10-09T04:55:08.187474+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.943483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":270,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232252+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:54:47.232252+0000","last_peered":"2023-10-09T04:54:47.232252+0000","last_clean":"2023-10-09T04:54:47.232252+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:54:47.232252+0000","last_undegraded":"2023-10-09T04:54:47.232252+0000","last_fullsized":"2023-10-09T04:54:47.232252+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":241,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231999+0000","last_change":"2023-10-09T04:51:14.108030+0000","last_active":"2023-10-09T04:54:47.231999+0000","last_peered":"2023-10-09T04:54:47.231999+0000","last_clean":"2023-10-09T04:54:47.231999+0000","last_became_active":"2023-10-09T04:51:14.107660+0000","last_became_peered":"2023-10-09T04:51:14.107660+0000","last_unstale":"2023-10-09T04:54:47.231999+0000","last_undegraded":"2023-10-09T04:54:47.231999+0000","last_fullsized":"2023-10-09T04:54:47.231999+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"308'7307","reported_seq":7330,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.663814+0000","last_change":"2023-10-09T04:53:24.137716+0000","last_active":"2023-10-09T04:55:28.663814+0000","last_peered":"2023-10-09T04:55:28.663814+0000","last_clean":"2023-10-09T04:55:28.663814+0000","last_became_active":"2023-10-09T04:53:24.137204+0000","last_became_peered":"2023-10-09T04:53:24.137204+0000","last_unstale":"2023-10-09T04:55:28.663814+0000","last_undegraded":"2023-10-09T04:55:28.663814+0000","last_fullsized":"2023-10-09T04:55:28.663814+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7307,"log_dups_size":0,"ondisk_log_size":7307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393019392,"num_objects":5997,"num_object_clones":0,"num_object_copies":17991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7307,"num_write_kb":425728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":769,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231963+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:54:47.231963+0000","last_peered":"2023-10-09T04:54:47.231963+0000","last_clean":"2023-10-09T04:54:47.231963+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:54:47.231963+0000","last_undegraded":"2023-10-09T04:54:47.231963+0000","last_fullsized":"2023-10-09T04:54:47.231963+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.190699+0000","last_change":"2023-10-09T04:51:14.112695+0000","last_active":"2023-10-09T04:55:26.190699+0000","last_peered":"2023-10-09T04:55:26.190699+0000","last_clean":"2023-10-09T04:55:26.190699+0000","last_became_active":"2023-10-09T04:51:14.112434+0000","last_became_peered":"2023-10-09T04:51:14.112434+0000","last_unstale":"2023-10-09T04:55:26.190699+0000","last_undegraded":"2023-10-09T04:55:26.190699+0000","last_fullsized":"2023-10-09T04:55:26.190699+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:10.034798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":254,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201090+0000","last_change":"2023-10-09T04:50:44.062081+0000","last_active":"2023-10-09T04:55:08.201090+0000","last_peered":"2023-10-09T04:55:08.201090+0000","last_clean":"2023-10-09T04:55:08.201090+0000","last_became_active":"2023-10-09T04:50:44.061730+0000","last_became_peered":"2023-10-09T04:50:44.061730+0000","last_unstale":"2023-10-09T04:55:08.201090+0000","last_undegraded":"2023-10-09T04:55:08.201090+0000","last_fullsized":"2023-10-09T04:55:08.201090+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:52.943977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"307'6981","reported_seq":7002,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:24.968815+0000","last_change":"2023-10-09T04:53:24.136930+0000","last_active":"2023-10-09T04:55:24.968815+0000","last_peered":"2023-10-09T04:55:24.968815+0000","last_clean":"2023-10-09T04:55:24.968815+0000","last_became_active":"2023-10-09T04:53:24.136546+0000","last_became_peered":"2023-10-09T04:53:24.136546+0000","last_unstale":"2023-10-09T04:55:24.968815+0000","last_undegraded":"2023-10-09T04:55:24.968815+0000","last_fullsized":"2023-10-09T04:55:24.968815+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401539072,"num_objects":6127,"num_object_clones":0,"num_object_copies":18381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6981,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1389,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.615267+0000","last_change":"2023-10-09T04:44:51.889738+0000","last_active":"2023-10-09T04:55:12.615267+0000","last_peered":"2023-10-09T04:55:12.615267+0000","last_clean":"2023-10-09T04:55:12.615267+0000","last_became_active":"2023-10-09T04:44:51.888765+0000","last_became_peered":"2023-10-09T04:44:51.888765+0000","last_unstale":"2023-10-09T04:55:12.615267+0000","last_undegraded":"2023-10-09T04:55:12.615267+0000","last_fullsized":"2023-10-09T04:55:12.615267+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:59.118212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":254,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.200374+0000","last_change":"2023-10-09T04:50:44.047494+0000","last_active":"2023-10-09T04:55:08.200374+0000","last_peered":"2023-10-09T04:55:08.200374+0000","last_clean":"2023-10-09T04:55:08.200374+0000","last_became_active":"2023-10-09T04:50:44.047111+0000","last_became_peered":"2023-10-09T04:50:44.047111+0000","last_unstale":"2023-10-09T04:55:08.200374+0000","last_undegraded":"2023-10-09T04:55:08.200374+0000","last_fullsized":"2023-10-09T04:55:08.200374+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:49.698697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232499+0000","last_change":"2023-10-09T04:50:44.051844+0000","last_active":"2023-10-09T04:54:47.232499+0000","last_peered":"2023-10-09T04:54:47.232499+0000","last_clean":"2023-10-09T04:54:47.232499+0000","last_became_active":"2023-10-09T04:50:44.051376+0000","last_became_peered":"2023-10-09T04:50:44.051376+0000","last_unstale":"2023-10-09T04:54:47.232499+0000","last_undegraded":"2023-10-09T04:54:47.232499+0000","last_fullsized":"2023-10-09T04:54:47.232499+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"308'7186","reported_seq":7209,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.648898+0000","last_change":"2023-10-09T04:53:24.352741+0000","last_active":"2023-10-09T04:55:28.648898+0000","last_peered":"2023-10-09T04:55:28.648898+0000","last_clean":"2023-10-09T04:55:28.648898+0000","last_became_active":"2023-10-09T04:53:24.351966+0000","last_became_peered":"2023-10-09T04:53:24.351966+0000","last_unstale":"2023-10-09T04:55:28.648898+0000","last_undegraded":"2023-10-09T04:55:28.648898+0000","last_fullsized":"2023-10-09T04:55:28.648898+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390463488,"num_objects":5958,"num_object_clones":0,"num_object_copies":17874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7186,"num_write_kb":420608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":832,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232076+0000","last_change":"2023-10-09T04:44:51.894121+0000","last_active":"2023-10-09T04:54:47.232076+0000","last_peered":"2023-10-09T04:54:47.232076+0000","last_clean":"2023-10-09T04:54:47.232076+0000","last_became_active":"2023-10-09T04:44:51.893520+0000","last_became_peered":"2023-10-09T04:44:51.893520+0000","last_unstale":"2023-10-09T04:54:47.232076+0000","last_undegraded":"2023-10-09T04:54:47.232076+0000","last_fullsized":"2023-10-09T04:54:47.232076+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":529,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201169+0000","last_change":"2023-10-09T04:40:48.818670+0000","last_active":"2023-10-09T04:55:08.201169+0000","last_peered":"2023-10-09T04:55:08.201169+0000","last_clean":"2023-10-09T04:55:08.201169+0000","last_became_active":"2023-10-09T04:40:48.818072+0000","last_became_peered":"2023-10-09T04:40:48.818072+0000","last_unstale":"2023-10-09T04:55:08.201169+0000","last_undegraded":"2023-10-09T04:55:08.201169+0000","last_fullsized":"2023-10-09T04:55:08.201169+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":225,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231865+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:54:47.231865+0000","last_peered":"2023-10-09T04:54:47.231865+0000","last_clean":"2023-10-09T04:54:47.231865+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:54:47.231865+0000","last_undegraded":"2023-10-09T04:54:47.231865+0000","last_fullsized":"2023-10-09T04:54:47.231865+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":251,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193160+0000","last_change":"2023-10-09T04:51:14.125228+0000","last_active":"2023-10-09T04:55:08.193160+0000","last_peered":"2023-10-09T04:55:08.193160+0000","last_clean":"2023-10-09T04:55:08.193160+0000","last_became_active":"2023-10-09T04:51:14.124848+0000","last_became_peered":"2023-10-09T04:51:14.124848+0000","last_unstale":"2023-10-09T04:55:08.193160+0000","last_undegraded":"2023-10-09T04:55:08.193160+0000","last_fullsized":"2023-10-09T04:55:08.193160+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"308'7084","reported_seq":7108,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:27.092540+0000","last_change":"2023-10-09T04:53:24.136377+0000","last_active":"2023-10-09T04:55:27.092540+0000","last_peered":"2023-10-09T04:55:27.092540+0000","last_clean":"2023-10-09T04:55:27.092540+0000","last_became_active":"2023-10-09T04:53:24.136061+0000","last_became_peered":"2023-10-09T04:53:24.136061+0000","last_unstale":"2023-10-09T04:55:27.092540+0000","last_undegraded":"2023-10-09T04:55:27.092540+0000","last_fullsized":"2023-10-09T04:55:27.092540+0000","mapping_epoch":293,"log_start":"301'100","ondisk_log_start":"301'100","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:09.217233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392953856,"num_objects":5996,"num_object_clones":0,"num_object_copies":17988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7084,"num_write_kb":418560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":695,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201175+0000","last_change":"2023-10-09T04:42:04.819909+0000","last_active":"2023-10-09T04:55:08.201175+0000","last_peered":"2023-10-09T04:55:08.201175+0000","last_clean":"2023-10-09T04:55:08.201175+0000","last_became_active":"2023-10-09T04:42:04.819143+0000","last_became_peered":"2023-10-09T04:42:04.819143+0000","last_unstale":"2023-10-09T04:55:08.201175+0000","last_undegraded":"2023-10-09T04:55:08.201175+0000","last_fullsized":"2023-10-09T04:55:08.201175+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":325,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.186175+0000","last_change":"2023-10-09T04:51:14.151803+0000","last_active":"2023-10-09T04:55:08.186175+0000","last_peered":"2023-10-09T04:55:08.186175+0000","last_clean":"2023-10-09T04:55:08.186175+0000","last_became_active":"2023-10-09T04:51:14.151334+0000","last_became_peered":"2023-10-09T04:51:14.151334+0000","last_unstale":"2023-10-09T04:55:08.186175+0000","last_undegraded":"2023-10-09T04:55:08.186175+0000","last_fullsized":"2023-10-09T04:55:08.186175+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:21.885966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":299,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.185616+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:55:08.185616+0000","last_peered":"2023-10-09T04:55:08.185616+0000","last_clean":"2023-10-09T04:55:08.185616+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:55:08.185616+0000","last_undegraded":"2023-10-09T04:55:08.185616+0000","last_fullsized":"2023-10-09T04:55:08.185616+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"308'7094","reported_seq":7118,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.632968+0000","last_change":"2023-10-09T04:53:24.140748+0000","last_active":"2023-10-09T04:55:28.632968+0000","last_peered":"2023-10-09T04:55:28.632968+0000","last_clean":"2023-10-09T04:55:28.632968+0000","last_became_active":"2023-10-09T04:53:24.140325+0000","last_became_peered":"2023-10-09T04:53:24.140325+0000","last_unstale":"2023-10-09T04:55:28.632968+0000","last_undegraded":"2023-10-09T04:55:28.632968+0000","last_fullsized":"2023-10-09T04:55:28.632968+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7094,"log_dups_size":0,"ondisk_log_size":7094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388169752,"num_objects":5924,"num_object_clones":0,"num_object_copies":17772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5924,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7094,"num_write_kb":416513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":566,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201516+0000","last_change":"2023-10-09T04:53:05.183235+0000","last_active":"2023-10-09T04:55:08.201516+0000","last_peered":"2023-10-09T04:55:08.201516+0000","last_clean":"2023-10-09T04:55:08.201516+0000","last_became_active":"2023-10-09T04:53:05.182472+0000","last_became_peered":"2023-10-09T04:53:05.182472+0000","last_unstale":"2023-10-09T04:55:08.201516+0000","last_undegraded":"2023-10-09T04:55:08.201516+0000","last_fullsized":"2023-10-09T04:55:08.201516+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":254,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201018+0000","last_change":"2023-10-09T04:50:44.052455+0000","last_active":"2023-10-09T04:55:08.201018+0000","last_peered":"2023-10-09T04:55:08.201018+0000","last_clean":"2023-10-09T04:55:08.201018+0000","last_became_active":"2023-10-09T04:50:44.052136+0000","last_became_peered":"2023-10-09T04:50:44.052136+0000","last_unstale":"2023-10-09T04:55:08.201018+0000","last_undegraded":"2023-10-09T04:55:08.201018+0000","last_fullsized":"2023-10-09T04:55:08.201018+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:32.549357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":287,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.189829+0000","last_change":"2023-10-09T04:40:47.823943+0000","last_active":"2023-10-09T04:55:08.189829+0000","last_peered":"2023-10-09T04:55:08.189829+0000","last_clean":"2023-10-09T04:55:08.189829+0000","last_became_active":"2023-10-09T04:40:47.817811+0000","last_became_peered":"2023-10-09T04:40:47.817811+0000","last_unstale":"2023-10-09T04:55:08.189829+0000","last_undegraded":"2023-10-09T04:55:08.189829+0000","last_fullsized":"2023-10-09T04:55:08.189829+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:07.959210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"308'7023","reported_seq":7045,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.653276+0000","last_change":"2023-10-09T04:53:24.137672+0000","last_active":"2023-10-09T04:55:28.653276+0000","last_peered":"2023-10-09T04:55:28.653276+0000","last_clean":"2023-10-09T04:55:28.653276+0000","last_became_active":"2023-10-09T04:53:24.137340+0000","last_became_peered":"2023-10-09T04:53:24.137340+0000","last_unstale":"2023-10-09T04:55:28.653276+0000","last_undegraded":"2023-10-09T04:55:28.653276+0000","last_fullsized":"2023-10-09T04:55:28.653276+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384368640,"num_objects":5865,"num_object_clones":0,"num_object_copies":17595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7023,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":127,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.202007+0000","last_change":"2023-10-09T04:40:48.359559+0000","last_active":"2023-10-09T04:55:08.202007+0000","last_peered":"2023-10-09T04:55:08.202007+0000","last_clean":"2023-10-09T04:55:08.202007+0000","last_became_active":"2023-10-09T04:40:47.830896+0000","last_became_peered":"2023-10-09T04:40:47.830896+0000","last_unstale":"2023-10-09T04:55:08.202007+0000","last_undegraded":"2023-10-09T04:55:08.202007+0000","last_fullsized":"2023-10-09T04:55:08.202007+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'91","reported_seq":368,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193245+0000","last_change":"2023-10-09T04:51:14.350042+0000","last_active":"2023-10-09T04:55:08.193245+0000","last_peered":"2023-10-09T04:55:08.193245+0000","last_clean":"2023-10-09T04:55:08.193245+0000","last_became_active":"2023-10-09T04:51:14.157185+0000","last_became_peered":"2023-10-09T04:51:14.157185+0000","last_unstale":"2023-10-09T04:55:08.193245+0000","last_undegraded":"2023-10-09T04:55:08.193245+0000","last_fullsized":"2023-10-09T04:55:08.193245+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":123,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.201042+0000","last_change":"2023-10-09T04:40:47.831780+0000","last_active":"2023-10-09T04:55:08.201042+0000","last_peered":"2023-10-09T04:55:08.201042+0000","last_clean":"2023-10-09T04:55:08.201042+0000","last_became_active":"2023-10-09T04:40:47.831435+0000","last_became_peered":"2023-10-09T04:40:47.831435+0000","last_unstale":"2023-10-09T04:55:08.201042+0000","last_undegraded":"2023-10-09T04:55:08.201042+0000","last_fullsized":"2023-10-09T04:55:08.201042+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"308'7297","reported_seq":7319,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.685090+0000","last_change":"2023-10-09T04:53:24.140212+0000","last_active":"2023-10-09T04:55:28.685090+0000","last_peered":"2023-10-09T04:55:28.685090+0000","last_clean":"2023-10-09T04:55:28.685090+0000","last_became_active":"2023-10-09T04:53:24.139824+0000","last_became_peered":"2023-10-09T04:53:24.139824+0000","last_unstale":"2023-10-09T04:55:28.685090+0000","last_undegraded":"2023-10-09T04:55:28.685090+0000","last_fullsized":"2023-10-09T04:55:28.685090+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":293,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:23.104083+0000","last_clean_scrub_stamp":"2023-10-09T04:53:23.104083+0000","objects_scrubbed":0,"log_size":7297,"log_dups_size":0,"ondisk_log_size":7297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399704064,"num_objects":6099,"num_object_clones":0,"num_object_copies":18297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7297,"num_write_kb":428672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":388,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.196618+0000","last_change":"2023-10-09T04:44:51.904173+0000","last_active":"2023-10-09T04:55:08.196618+0000","last_peered":"2023-10-09T04:55:08.196618+0000","last_clean":"2023-10-09T04:55:08.196618+0000","last_became_active":"2023-10-09T04:44:51.903435+0000","last_became_peered":"2023-10-09T04:44:51.903435+0000","last_unstale":"2023-10-09T04:55:08.196618+0000","last_undegraded":"2023-10-09T04:55:08.196618+0000","last_fullsized":"2023-10-09T04:55:08.196618+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":286,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.185706+0000","last_change":"2023-10-09T04:51:14.178205+0000","last_active":"2023-10-09T04:55:08.185706+0000","last_peered":"2023-10-09T04:55:08.185706+0000","last_clean":"2023-10-09T04:55:08.185706+0000","last_became_active":"2023-10-09T04:51:14.147035+0000","last_became_peered":"2023-10-09T04:51:14.147035+0000","last_unstale":"2023-10-09T04:55:08.185706+0000","last_undegraded":"2023-10-09T04:55:08.185706+0000","last_fullsized":"2023-10-09T04:55:08.185706+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:04.219571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231946+0000","last_change":"2023-10-09T04:51:14.110047+0000","last_active":"2023-10-09T04:54:47.231946+0000","last_peered":"2023-10-09T04:54:47.231946+0000","last_clean":"2023-10-09T04:54:47.231946+0000","last_became_active":"2023-10-09T04:51:14.109658+0000","last_became_peered":"2023-10-09T04:51:14.109658+0000","last_unstale":"2023-10-09T04:54:47.231946+0000","last_undegraded":"2023-10-09T04:54:47.231946+0000","last_fullsized":"2023-10-09T04:54:47.231946+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":283,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.189785+0000","last_change":"2023-10-09T04:51:14.148515+0000","last_active":"2023-10-09T04:55:08.189785+0000","last_peered":"2023-10-09T04:55:08.189785+0000","last_clean":"2023-10-09T04:55:08.189785+0000","last_became_active":"2023-10-09T04:51:14.147834+0000","last_became_peered":"2023-10-09T04:51:14.147834+0000","last_unstale":"2023-10-09T04:55:08.189785+0000","last_undegraded":"2023-10-09T04:55:08.189785+0000","last_fullsized":"2023-10-09T04:55:08.189785+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:48.693686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.200828+0000","last_change":"2023-10-09T04:44:51.889670+0000","last_active":"2023-10-09T04:55:08.200828+0000","last_peered":"2023-10-09T04:55:08.200828+0000","last_clean":"2023-10-09T04:55:08.200828+0000","last_became_active":"2023-10-09T04:44:51.889343+0000","last_became_peered":"2023-10-09T04:44:51.889343+0000","last_unstale":"2023-10-09T04:55:08.200828+0000","last_undegraded":"2023-10-09T04:55:08.200828+0000","last_fullsized":"2023-10-09T04:55:08.200828+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:17.765323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":245,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232073+0000","last_change":"2023-10-09T04:51:14.120655+0000","last_active":"2023-10-09T04:54:47.232073+0000","last_peered":"2023-10-09T04:54:47.232073+0000","last_clean":"2023-10-09T04:54:47.232073+0000","last_became_active":"2023-10-09T04:51:14.109662+0000","last_became_peered":"2023-10-09T04:51:14.109662+0000","last_unstale":"2023-10-09T04:54:47.232073+0000","last_undegraded":"2023-10-09T04:54:47.232073+0000","last_fullsized":"2023-10-09T04:54:47.232073+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":40,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.200496+0000","last_change":"2023-10-09T04:51:14.152593+0000","last_active":"2023-10-09T04:55:08.200496+0000","last_peered":"2023-10-09T04:55:08.200496+0000","last_clean":"2023-10-09T04:55:08.200496+0000","last_became_active":"2023-10-09T04:51:14.152155+0000","last_became_peered":"2023-10-09T04:51:14.152155+0000","last_unstale":"2023-10-09T04:55:08.200496+0000","last_undegraded":"2023-10-09T04:55:08.200496+0000","last_fullsized":"2023-10-09T04:55:08.200496+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:51.059337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":238,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.232251+0000","last_change":"2023-10-09T04:51:14.164283+0000","last_active":"2023-10-09T04:54:47.232251+0000","last_peered":"2023-10-09T04:54:47.232251+0000","last_clean":"2023-10-09T04:54:47.232251+0000","last_became_active":"2023-10-09T04:51:14.111363+0000","last_became_peered":"2023-10-09T04:51:14.111363+0000","last_unstale":"2023-10-09T04:54:47.232251+0000","last_undegraded":"2023-10-09T04:54:47.232251+0000","last_fullsized":"2023-10-09T04:54:47.232251+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":231,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.190996+0000","last_change":"2023-10-09T04:51:14.150509+0000","last_active":"2023-10-09T04:55:08.190996+0000","last_peered":"2023-10-09T04:55:08.190996+0000","last_clean":"2023-10-09T04:55:08.190996+0000","last_became_active":"2023-10-09T04:51:14.150128+0000","last_became_peered":"2023-10-09T04:51:14.150128+0000","last_unstale":"2023-10-09T04:55:08.190996+0000","last_undegraded":"2023-10-09T04:55:08.190996+0000","last_fullsized":"2023-10-09T04:55:08.190996+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:17.197700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":209,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.193174+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:55:08.193174+0000","last_peered":"2023-10-09T04:55:08.193174+0000","last_clean":"2023-10-09T04:55:08.193174+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:55:08.193174+0000","last_undegraded":"2023-10-09T04:55:08.193174+0000","last_fullsized":"2023-10-09T04:55:08.193174+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T04:54:47.231828+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T04:54:47.231828+0000","last_peered":"2023-10-09T04:54:47.231828+0000","last_clean":"2023-10-09T04:54:47.231828+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T04:54:47.231828+0000","last_undegraded":"2023-10-09T04:54:47.231828+0000","last_fullsized":"2023-10-09T04:54:47.231828+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":303,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.186168+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:55:08.186168+0000","last_peered":"2023-10-09T04:55:08.186168+0000","last_clean":"2023-10-09T04:55:08.186168+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:55:08.186168+0000","last_undegraded":"2023-10-09T04:55:08.186168+0000","last_fullsized":"2023-10-09T04:55:08.186168+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T04:55:08.191295+0000","last_change":"2023-10-09T04:51:14.146277+0000","last_active":"2023-10-09T04:55:08.191295+0000","last_peered":"2023-10-09T04:55:08.191295+0000","last_clean":"2023-10-09T04:55:08.191295+0000","last_became_active":"2023-10-09T04:51:14.145974+0000","last_became_peered":"2023-10-09T04:51:14.145974+0000","last_unstale":"2023-10-09T04:55:08.191295+0000","last_undegraded":"2023-10-09T04:55:08.191295+0000","last_fullsized":"2023-10-09T04:55:08.191295+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6329532440,"num_objects":96582,"num_object_clones":0,"num_object_copies":289746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96582,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114312,"num_write_kb":6748545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19098185728,"data_stored":19098173512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114012,"ondisk_log_size":114012,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":9830,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430772,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2856100,"kb_used_data":1956572,"kb_used_omap":0,"kb_used_meta":899520,"kb_avail":90889052,"statfs":{"total":95995035648,"available":93070389248,"internally_reserved":0,"allocated":2003529728,"data_stored":2002925529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921108480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:27 2023","interfaces":[{"interface":"back","average":{"1min":30.649000000000001,"5min":9.5790000000000006,"15min":4.8520000000000003},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":422.35000000000002,"5min":422.35000000000002,"15min":422.35000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":32.039000000000001,"5min":9.827,"15min":4.8259999999999996},"min":{"1min":0.38600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":423.267,"5min":423.267,"15min":423.267},"last":3.859}]},{"osd":1,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":2.8319999999999999,"15min":2.3050000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":46.222999999999999,"5min":57.668999999999997,"15min":58.816000000000003},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.016,"5min":2.895,"15min":2.2589999999999999},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":45.536000000000001,"5min":58.046999999999997,"15min":58.840000000000003},"last":0.80100000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:55:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.569,"15min":1.6319999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.314},"max":{"1min":5.3600000000000003,"5min":13.138999999999999,"15min":13.138999999999999},"last":4.3860000000000001},{"interface":"front","average":{"1min":1.425,"5min":1.3049999999999999,"15min":1.857},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":6.4349999999999996,"5min":6.7809999999999997,"15min":11.583},"last":0.84299999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.6100000000000001,"15min":2.069},"min":{"1min":0.42399999999999999,"5min":0.374,"15min":0.26700000000000002},"max":{"1min":2.254,"5min":8.6449999999999996,"15min":102.02800000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.752,"5min":1.7030000000000001,"15min":2.0230000000000001},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.32100000000000001},"max":{"1min":2.101,"5min":13.195,"15min":101.973},"last":4.298}]},{"osd":4,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.294,"15min":1.3029999999999999},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.012,"5min":10.199,"15min":11.933999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.77600000000000002,"5min":1.2470000000000001,"15min":1.341},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.308},"max":{"1min":2.2930000000000001,"5min":8.1080000000000005,"15min":8.452},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:55:19 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":1.5169999999999999,"15min":1.492},"min":{"1min":0.39500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.1499999999999999,"5min":8.6050000000000004,"15min":11.268000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.874,"5min":1.6040000000000001,"15min":1.6319999999999999},"min":{"1min":0.42299999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":3.3330000000000002,"5min":9.1449999999999996,"15min":15.138999999999999},"last":0.61399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.173,"5min":1.6899999999999999,"15min":1.472},"min":{"1min":0.442,"5min":0.40400000000000003,"15min":0.36899999999999999},"max":{"1min":27.059000000000001,"5min":27.059000000000001,"15min":27.059000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.77,"15min":1.552},"min":{"1min":0.44,"5min":0.39300000000000002,"15min":0.36599999999999999},"max":{"1min":27.698,"5min":27.698,"15min":27.698},"last":4.3479999999999999}]}]},{"osd":6,"up_from":146,"seq":627065225473,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3314924,"kb_used_data":2696484,"kb_used_omap":5,"kb_used_meta":618426,"kb_avail":90430228,"statfs":{"total":95995035648,"available":92600553472,"internally_reserved":0,"allocated":2761199616,"data_stored":2760594996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":633268832},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":6.4429999999999996,"5min":2.996,"15min":3.4580000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":81.287999999999997,"5min":81.287999999999997,"15min":214.82300000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":6.2370000000000001,"5min":2.4689999999999999,"15min":3.2799999999999998},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":81.168999999999997,"5min":81.168999999999997,"15min":213.34700000000001},"last":0.312}]},{"osd":1,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.5409999999999999,"15min":2.6469999999999998},"min":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":9.9570000000000007,"5min":40.709000000000003,"15min":76.081000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.5369999999999999,"15min":2.3900000000000001},"min":{"1min":0.39400000000000002,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":5.742,"5min":40.414000000000001,"15min":75.546000000000006},"last":0.45700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":1.2709999999999999,"15min":1.607},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6890000000000001,"5min":5.8540000000000001,"15min":6.8940000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.2250000000000001,"5min":0.97099999999999997,"15min":1.2769999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.311},"max":{"1min":4.6669999999999998,"5min":5.0709999999999997,"15min":9.4290000000000003},"last":0.66600000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":1.6240000000000001,"15min":1.5840000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":12.050000000000001,"5min":12.050000000000001,"15min":12.050000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.883,"5min":1.5960000000000001,"15min":1.48},"min":{"1min":0.41999999999999998,"5min":0.34499999999999997,"15min":0.32600000000000001},"max":{"1min":5.1200000000000001,"5min":8.9079999999999995,"15min":9.3190000000000008},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.88,"15min":1.1160000000000001},"min":{"1min":0.38900000000000001,"5min":0.32800000000000001,"15min":0.248},"max":{"1min":4.7069999999999999,"5min":4.7069999999999999,"15min":13.699999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.657,"5min":1.3779999999999999,"15min":1.3939999999999999},"min":{"1min":0.36599999999999999,"5min":0.35899999999999999,"15min":0.307},"max":{"1min":7.476,"5min":10.428000000000001,"15min":10.428000000000001},"last":0.34499999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:55:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.3400000000000001,"15min":1.3720000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":29.077999999999999,"5min":29.077999999999999,"15min":29.077999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":2.6000000000000001,"5min":1.988,"15min":1.768},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.29599999999999999},"max":{"1min":28.742000000000001,"5min":28.742000000000001,"15min":28.742000000000001},"last":0.375}]},{"osd":7,"last update":"Mon Oct 9 04:54:53 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.6719999999999999,"15min":1.6180000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.33000000000000002},"max":{"1min":10.526999999999999,"5min":10.701000000000001,"15min":16.527999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.4530000000000001,"15min":1.4730000000000001},"min":{"1min":0.45900000000000002,"5min":0.41699999999999998,"15min":0.33300000000000002},"max":{"1min":4.7939999999999996,"5min":6.6180000000000003,"15min":15.366},"last":0.53900000000000003}]}]},{"osd":1,"up_from":93,"seq":399431958922,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3163660,"kb_used_data":2300356,"kb_used_omap":6,"kb_used_meta":863289,"kb_avail":90581492,"statfs":{"total":95995035648,"available":92755447808,"internally_reserved":0,"allocated":2355564544,"data_stored":2354407584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":884007948},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.2389999999999999,"15min":2.117},"min":{"1min":0.22800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":47.274000000000001,"5min":47.274000000000001,"15min":47.274000000000001},"last":7.9279999999999999},{"interface":"front","average":{"1min":4.3600000000000003,"5min":1.8939999999999999,"15min":2.04},"min":{"1min":0.36799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":47.372999999999998,"5min":47.372999999999998,"15min":47.372999999999998},"last":0.55100000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:55:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.361,"15min":1.9430000000000001},"min":{"1min":0.27600000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.1980000000000004,"5min":8.2650000000000006,"15min":15.336},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.194,"15min":1.7769999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.494,"5min":10.494,"15min":13.444000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:55:02 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":2.0030000000000001,"15min":1.827},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20999999999999999},"max":{"1min":10.589,"5min":40.210000000000001,"15min":40.210000000000001},"last":5.3760000000000003},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.0070000000000001,"15min":1.8},"min":{"1min":0.442,"5min":0.26000000000000001,"15min":0.20200000000000001},"max":{"1min":10.574,"5min":40.127000000000002,"15min":40.127000000000002},"last":0.68899999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.387,"15min":1.458},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.29199999999999998},"max":{"1min":4.1769999999999996,"5min":7.923,"15min":13.362},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.752,"5min":1.627,"15min":1.599},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.316},"max":{"1min":9.6259999999999994,"5min":9.827,"15min":10.923999999999999},"last":5.4249999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:55:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.5580000000000001,"15min":1.649},"min":{"1min":0.374,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1039999999999992,"5min":9.2910000000000004,"15min":14.494999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.579,"15min":1.671},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":10.435,"5min":10.435,"15min":13.413},"last":0.60399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2810000000000001,"5min":1.7549999999999999,"15min":1.7649999999999999},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.32600000000000001},"max":{"1min":18.295000000000002,"5min":18.295000000000002,"15min":18.295000000000002},"last":3.9420000000000002},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.7869999999999999,"15min":1.841},"min":{"1min":0.48699999999999999,"5min":0.41999999999999998,"15min":0.40500000000000003},"max":{"1min":18.34,"5min":18.34,"15min":18.34},"last":3.98}]},{"osd":7,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.5489999999999999,"15min":1.673},"min":{"1min":0.40699999999999997,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":4.1479999999999997,"5min":7.8899999999999997,"15min":13.474},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.623,"5min":1.889,"15min":1.7909999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.36399999999999999},"max":{"1min":9.5909999999999993,"5min":9.8079999999999998,"15min":18.074000000000002},"last":5.46}]}]},{"osd":0,"up_from":88,"seq":377957122447,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4780692,"kb_used_data":3901196,"kb_used_omap":9,"kb_used_meta":879350,"kb_avail":88964460,"statfs":{"total":95995035648,"available":91099607040,"internally_reserved":0,"allocated":3994824704,"data_stored":3993663773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":900454897},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,6,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":2.0680000000000001,"15min":1.585},"min":{"1min":0.254,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":5.2949999999999999,"5min":62.396000000000001,"15min":62.396000000000001},"last":2.5800000000000001},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.2280000000000002,"15min":2.0289999999999999},"min":{"1min":0.25900000000000001,"5min":0.19900000000000001,"15min":0.19400000000000001},"max":{"1min":6.6180000000000003,"5min":62.496000000000002,"15min":62.496000000000002},"last":2.5539999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":1.1160000000000001,"15min":1.788},"min":{"1min":0.32300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.0830000000000002,"5min":8.0830000000000002,"15min":17.109000000000002},"last":2.7309999999999999},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.365,"15min":2.1030000000000002},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.25600000000000001},"max":{"1min":6.2290000000000001,"5min":6.3929999999999998,"15min":12.57},"last":2.7160000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":1.4410000000000001,"15min":1.4490000000000001},"min":{"1min":0.35699999999999998,"5min":0.216,"15min":0.216},"max":{"1min":8.1150000000000002,"5min":8.1150000000000002,"15min":12.333},"last":2.6459999999999999},{"interface":"front","average":{"1min":1.988,"5min":1.6899999999999999,"15min":1.792},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.23999999999999999},"max":{"1min":6.5720000000000001,"5min":8.1029999999999998,"15min":14.956},"last":3.1219999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:55:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.8,"15min":1.6739999999999999},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.27500000000000002},"max":{"1min":8.2959999999999994,"5min":9.6509999999999998,"15min":16.84},"last":2.661},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.4339999999999999,"15min":1.415},"min":{"1min":0.51400000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.0549999999999997,"5min":8.0549999999999997,"15min":39.957999999999998},"last":3.1349999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.635,"15min":1.6499999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.35399999999999998},"max":{"1min":7.2889999999999997,"5min":7.2889999999999997,"15min":11.617000000000001},"last":3.1760000000000002},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.671,"15min":1.5089999999999999},"min":{"1min":0.51300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.404,"5min":10.404,"15min":10.404},"last":3.2440000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.157,"15min":1.3779999999999999},"min":{"1min":0.45900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.9320000000000004,"5min":6.9320000000000004,"15min":10.345000000000001},"last":3.1099999999999999},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.6519999999999999,"15min":1.7529999999999999},"min":{"1min":0.48699999999999999,"5min":0.41199999999999998,"15min":0.39400000000000002},"max":{"1min":7.7009999999999996,"5min":7.7009999999999996,"15min":11.57},"last":3.1520000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:55:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.2040000000000002,"15min":1.966},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":13.518000000000001,"5min":43.276000000000003,"15min":43.276000000000003},"last":3.198},{"interface":"front","average":{"1min":1.893,"5min":2.484,"15min":1.9750000000000001},"min":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.35499999999999998},"max":{"1min":4.2629999999999999,"5min":46.064,"15min":46.064},"last":2.6190000000000002}]}]},{"osd":2,"up_from":234,"seq":1005022347392,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2620012,"kb_used_data":2336356,"kb_used_omap":13,"kb_used_meta":283634,"kb_avail":91125140,"statfs":{"total":95995035648,"available":93312143360,"internally_reserved":0,"allocated":2392428544,"data_stored":2391246626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":290442111},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":3.4769999999999999,"15min":2.806},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":6.0659999999999998,"5min":112.312,"15min":112.312},"last":5.4009999999999998},{"interface":"front","average":{"1min":1.6910000000000001,"5min":3.5720000000000001,"15min":2.7999999999999998},"min":{"1min":0.245,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.7519999999999998,"5min":117.881,"15min":117.881},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":8.0589999999999993,"15min":4.6820000000000004},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.5890000000000004,"5min":267.54000000000002,"15min":267.54000000000002},"last":5.1920000000000002},{"interface":"front","average":{"1min":1.659,"5min":8.1669999999999998,"15min":4.4249999999999998},"min":{"1min":0.27300000000000002,"5min":0.252,"15min":0.22900000000000001},"max":{"1min":5.8200000000000003,"5min":268.12200000000001,"15min":268.12200000000001},"last":0.72999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.474,"15min":1.4950000000000001},"min":{"1min":0.251,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.747999999999999,"5min":14.334,"15min":14.340999999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.569,"15min":2.0499999999999998},"min":{"1min":0.34000000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.532,"5min":13.167999999999999,"15min":13.167999999999999},"last":1.0680000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.2370000000000001,"15min":1.667},"min":{"1min":0.42899999999999999,"5min":0.35399999999999998,"15min":0.34599999999999997},"max":{"1min":5.9219999999999997,"5min":10.051,"15min":10.051},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.079,"15min":1.5529999999999999},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":5.6470000000000002,"5min":10.119999999999999,"15min":10.119999999999999},"last":1.9510000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.2929999999999999,"15min":1.661},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.8869999999999996,"5min":10.856999999999999,"15min":10.856999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.407,"15min":2.117},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.316},"max":{"1min":6.0389999999999997,"5min":11.75,"15min":12.295999999999999},"last":1.04}]},{"osd":6,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":1.351,"15min":1.748},"min":{"1min":0.438,"5min":0.371,"15min":0.371},"max":{"1min":5.7249999999999996,"5min":10.092000000000001,"15min":10.092000000000001},"last":1.9850000000000001},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.3140000000000001,"15min":1.4790000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.35599999999999998},"max":{"1min":5.9130000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.754}]},{"osd":7,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.5289999999999999,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.8410000000000002,"5min":10.007999999999999,"15min":14.161},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.3500000000000001,"15min":1.6950000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.41199999999999998},"max":{"1min":5.8029999999999999,"5min":9.8550000000000004,"15min":10.996},"last":2.0089999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631867,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4074524,"kb_used_data":3192596,"kb_used_omap":5,"kb_used_meta":881786,"kb_avail":89670628,"statfs":{"total":95995035648,"available":91822723072,"internally_reserved":0,"allocated":3269218304,"data_stored":3268041278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":902949038},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":4.5439999999999996,"15min":2.98},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19500000000000001},"max":{"1min":6.6509999999999998,"5min":144.625,"15min":144.625},"last":3.7639999999999998},{"interface":"front","average":{"1min":2.0619999999999998,"5min":4.6230000000000002,"15min":2.9980000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.20000000000000001},"max":{"1min":9.0579999999999998,"5min":144.24100000000001,"15min":144.24100000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":4.0810000000000004,"15min":2.5070000000000001},"min":{"1min":0.23599999999999999,"5min":0.214,"15min":0.20100000000000001},"max":{"1min":5.1550000000000002,"5min":143.34100000000001,"15min":143.34100000000001},"last":5.3310000000000004},{"interface":"front","average":{"1min":2.0449999999999999,"5min":4.3810000000000002,"15min":2.7599999999999998},"min":{"1min":0.29599999999999999,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":6.6909999999999998,"5min":149.15700000000001,"15min":149.15700000000001},"last":4.2050000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:55:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.4099999999999999,"15min":1.492},"min":{"1min":0.26300000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":8.9900000000000002,"5min":10.148,"15min":10.148},"last":7.407},{"interface":"front","average":{"1min":3.3420000000000001,"5min":1.6879999999999999,"15min":1.696},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.619999999999999,"5min":15.619999999999999,"15min":15.619999999999999},"last":0.68600000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.3169999999999999,"15min":1.365},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":5.5300000000000002,"5min":5.5300000000000002,"15min":8.8420000000000005},"last":3.9060000000000001},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.2549999999999999,"15min":1.4059999999999999},"min":{"1min":0.39200000000000002,"5min":0.32000000000000001,"15min":0.29199999999999998},"max":{"1min":5.359,"5min":8.8840000000000003,"15min":9.4359999999999999},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:55:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6739999999999999,"5min":2.1499999999999999,"15min":1.9379999999999999},"min":{"1min":0.47699999999999998,"5min":0.39900000000000002,"15min":0.34799999999999998},"max":{"1min":23.073,"5min":23.073,"15min":23.073},"last":0.64900000000000002},{"interface":"front","average":{"1min":3.8620000000000001,"5min":2.1859999999999999,"15min":1.7190000000000001},"min":{"1min":0.45700000000000002,"5min":0.379,"15min":0.34200000000000003},"max":{"1min":22.832000000000001,"5min":22.832000000000001,"15min":22.832000000000001},"last":0.502}]},{"osd":6,"last update":"Mon Oct 9 04:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.7010000000000001,"15min":1.4990000000000001},"min":{"1min":0.40600000000000003,"5min":0.36799999999999999,"15min":0.32100000000000001},"max":{"1min":5.6749999999999998,"5min":10.102,"15min":10.102},"last":4.9889999999999999},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.335,"15min":1.3620000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35399999999999998},"max":{"1min":5.4939999999999998,"5min":5.4939999999999998,"15min":8.8699999999999992},"last":3.8820000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.7450000000000001,"15min":1.6759999999999999},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.33100000000000002},"max":{"1min":5.4630000000000001,"5min":15.170999999999999,"15min":15.170999999999999},"last":5.0170000000000003},{"interface":"front","average":{"1min":1.694,"5min":1.554,"15min":1.5760000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.36699999999999999},"max":{"1min":5.407,"5min":6.3019999999999996,"15min":9.2370000000000001},"last":3.8170000000000002}]}]},{"osd":4,"up_from":0,"seq":128849019469,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":150,"seq":644245094651,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3333196,"kb_used_data":2373252,"kb_used_omap":6,"kb_used_meta":959929,"kb_avail":90411956,"statfs":{"total":95995035648,"available":92581842944,"internally_reserved":0,"allocated":2430210048,"data_stored":2429625259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":982967571},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":17.763999999999999,"5min":7.6349999999999998,"15min":3.7589999999999999},"min":{"1min":0.38400000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":363.64499999999998,"5min":363.64499999999998,"15min":363.64499999999998},"last":1.0640000000000001},{"interface":"front","average":{"1min":17.446999999999999,"5min":8.0510000000000002,"15min":4.0620000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30099999999999999},"max":{"1min":363.26900000000001,"5min":363.26900000000001,"15min":363.26900000000001},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":5.2009999999999996,"15min":2.903},"min":{"1min":0.308,"5min":0.308,"15min":0.28399999999999997},"max":{"1min":3.9820000000000002,"5min":130.762,"15min":130.762},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.044,"5min":5.6680000000000001,"15min":3.2480000000000002},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.26400000000000001},"max":{"1min":4.5279999999999996,"5min":130.86199999999999,"15min":130.86199999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":10.971,"5min":3.0600000000000001,"15min":2.0430000000000001},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":208.40100000000001,"5min":208.40100000000001,"15min":208.40100000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":11.106,"5min":3.3959999999999999,"15min":2.4660000000000002},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":208.078,"5min":208.078,"15min":208.078},"last":0.59799999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":3.2080000000000002,"15min":2.3439999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.5720000000000001,"5min":130.69200000000001,"15min":130.69200000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.286,"5min":3.2879999999999998,"15min":2.5179999999999998},"min":{"1min":0.443,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.5750000000000002,"5min":130.53399999999999,"15min":130.53399999999999},"last":0.35399999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.345,"15min":1.417},"min":{"1min":0.40200000000000002,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":1.702,"5min":12.734,"15min":12.734},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.115,"15min":1.262},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":1.484,"5min":8.3409999999999993,"15min":10.709},"last":0.46500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.036,"15min":1.296},"min":{"1min":0.45200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.5820000000000001,"5min":7.2489999999999997,"15min":10.77},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.595,"15min":1.4990000000000001},"min":{"1min":0.41299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4409999999999998,"5min":14.223000000000001,"15min":14.223000000000001},"last":0.64400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":2.1579999999999999,"15min":1.7909999999999999},"min":{"1min":0.46600000000000003,"5min":0.42499999999999999,"15min":0.32200000000000001},"max":{"1min":2.8250000000000002,"5min":12.706,"15min":12.706},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.012,"5min":1.7050000000000001,"15min":1.651},"min":{"1min":0.39600000000000002,"5min":0.39200000000000002,"15min":0.35699999999999998},"max":{"1min":2.3639999999999999,"5min":21.797000000000001,"15min":21.797000000000001},"last":0.433}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3970306048,"data_stored":3970301976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2347433984,"data_stored":2347433984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2373189632,"data_stored":2373189632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3264024576,"data_stored":3264020504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2417688576,"data_stored":2417688576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2736066560,"data_stored":2736062488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1989476352,"data_stored":1989476352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:55:31.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-10-09T04:55:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.803 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:31.934 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:31.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:32.043 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:32.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:32.189 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:32.216 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:32.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.329 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:32 smithi082 bash[77063]: audit 2023-10-09T04:55:30.559727+0000 mgr.y (mgr.34107) 1247 : audit 0 from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:32.330 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:32 smithi082 bash[77063]: cluster 2023-10-09T04:55:30.874015+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T04:55:32.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:32 smithi082 bash[79733]: audit 2023-10-09T04:55:30.559727+0000 mgr.y (mgr.34107) 1247 : audit 0 from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:32.330 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:32 smithi082 bash[79733]: cluster 2023-10-09T04:55:30.874015+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T04:55:32.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:32.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:32.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:32 smithi153 bash[59565]: audit 2023-10-09T04:55:30.559727+0000 mgr.y (mgr.34107) 1247 : audit 0 from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:32 smithi153 bash[59565]: cluster 2023-10-09T04:55:30.874015+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T04:55:32.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:32.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:32.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:32.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:32.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:32.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:33.003 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:33.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:33.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:33.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:33.168 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:33.272 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:33.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:33.452 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:33.481 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:33.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:33.622 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:33.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:33.758 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:33.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:33.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:33.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:33.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:33.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:34.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:34.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:34.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:34.279 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.296 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:34 smithi082 bash[77063]: cluster 2023-10-09T04:55:32.874613+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T04:55:34.297 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:34 smithi082 bash[79733]: cluster 2023-10-09T04:55:32.874613+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T04:55:34.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:34.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.448 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:34.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.595 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:34 smithi153 bash[59565]: cluster 2023-10-09T04:55:32.874613+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T04:55:34.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:34.829 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:34.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:34.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:35.095 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:35.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:35.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:35.274 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:35.316 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:35.316 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:35.378 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:35.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:35.508 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:35.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:35.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:35.653 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:35.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:35.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:35.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:35.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:35.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:35.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:36.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:36.215 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:36.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:36.259 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:36.335 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:36.349 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:36.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:36.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:36.531 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:36.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.646 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:36 smithi082 bash[77063]: cluster 2023-10-09T04:55:34.875098+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s 2023-10-09T04:55:36.647 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:36 smithi082 bash[79733]: cluster 2023-10-09T04:55:34.875098+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s 2023-10-09T04:55:36.662 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:36 smithi153 bash[59565]: cluster 2023-10-09T04:55:34.875098+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 772 op/s 2023-10-09T04:55:36.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:36.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:36.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:36.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:36.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:37.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:37.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:37.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:37.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:37.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:37.460 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:37.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:37.621 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:37.725 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:37.849 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:37.880 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:37.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:37.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:38.010 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:38.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:38.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:38.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:38.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:38.291 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:38.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:38.404 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:38 smithi153 bash[59565]: cluster 2023-10-09T04:55:36.875543+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:55:38.419 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:38.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:38.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:38 smithi082 bash[77063]: cluster 2023-10-09T04:55:36.875543+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:55:38.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:38 smithi082 bash[79733]: cluster 2023-10-09T04:55:36.875543+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1371: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:55:38.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:38.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:38.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:38.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:38.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:38.820 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:38.924 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:38.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:38.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:39.051 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:39.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:39.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:39.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:39.213 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:39.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:39.317 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:39.357 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:39.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:39.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:39.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:39.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:39.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:39.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:39.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:39] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.33.4" 2023-10-09T04:55:39.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:39.854 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:39.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:39.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:39.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.017 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:40.119 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:40.145 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:40.205 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:40 smithi082 bash[77063]: cluster 2023-10-09T04:55:38.876120+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:55:40.248 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:40.283 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:40 smithi082 bash[79733]: cluster 2023-10-09T04:55:38.876120+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:55:40.384 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:40.392 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:40 smithi153 bash[59565]: cluster 2023-10-09T04:55:38.876120+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:55:40.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:40.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:40.535 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:40.643 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:40.787 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:40.823 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:40.926 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:40.958 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:40.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:41.065 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:41.094 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:41.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:41.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:41.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:41.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:41.602 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:41.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:41.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:41.766 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:41.788 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:55:41.788 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:55:41.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.899 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:41.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:42.003 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:42.028 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:42.135 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:42.167 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:42.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:42.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.280 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:42 smithi153 bash[59565]: cluster 2023-10-09T04:55:40.876733+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T04:55:42.306 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:42.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.437 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:42.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:42 smithi082 bash[77063]: cluster 2023-10-09T04:55:40.876733+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T04:55:42.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:42 smithi082 bash[79733]: cluster 2023-10-09T04:55:40.876733+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T04:55:42.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:42.570 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:42.676 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:42.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:42.809 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.811 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:55:42.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:42.947 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:42.977 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:42.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:43.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:43.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:43.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:43.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:43.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:43.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.524 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:43.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.663 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:43.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:43.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:43.805 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:43.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:43.941 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:43.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:43.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:44.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:44.066 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:44.173 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:44.206 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:44.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:44.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:44 smithi153 bash[59565]: cluster 2023-10-09T04:55:42.877360+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T04:55:44.337 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:44.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:44.452 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:44 smithi082 bash[77063]: cluster 2023-10-09T04:55:42.877360+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T04:55:44.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:44 smithi082 bash[79733]: cluster 2023-10-09T04:55:42.877360+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1374: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T04:55:44.467 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:44.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:44.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:44.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:44.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:44.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:44.866 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:44.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:44.974 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:45.020 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:45.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:45.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:45.263 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:45.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:45.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:45 smithi082 bash[77063]: audit 2023-10-09T04:55:44.299688+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:45.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:45 smithi082 bash[79733]: audit 2023-10-09T04:55:44.299688+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:45.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:45.537 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:45.546 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:45 smithi153 bash[59565]: audit 2023-10-09T04:55:44.299688+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:45.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:45.665 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:45.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:45.924 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:45.947 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:45.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:46.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:46.077 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:46.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:46.226 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:46.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:46.359 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:46.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:46.495 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:46.497 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:46 smithi153 bash[59565]: cluster 2023-10-09T04:55:44.877872+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:55:46.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:46.537 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:46 smithi082 bash[77063]: cluster 2023-10-09T04:55:44.877872+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:55:46.538 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:46 smithi082 bash[79733]: cluster 2023-10-09T04:55:44.877872+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1375: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:55:46.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:46.639 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:46.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:46.772 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:46.875 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:46.905 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:46.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:47.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:47.142 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:47.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.312 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:47.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.441 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:47.544 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:47 smithi082 bash[77063]: cluster 2023-10-09T04:55:46.878494+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T04:55:47.545 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:47 smithi082 bash[79733]: cluster 2023-10-09T04:55:46.878494+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T04:55:47.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:47 smithi153 bash[59565]: cluster 2023-10-09T04:55:46.878494+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T04:55:47.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:47.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:47.673 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:47.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:47.716 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:47.820 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:47.849 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:47.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:47.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:48.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:48.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:48.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:48.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:48.343 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:55:48.352 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:48.380 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:48.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:48.388 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:55:48.433 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:55:48.481 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:55:48.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:48.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:48.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:48.526 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:55:48.561 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:55:48.599 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:55:48.622 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:48.631 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:48 smithi082 bash[77063]: audit 2023-10-09T04:55:48.278915+0000 mon.a (mon.0) 1816 : audit 0 from='client.? 172.21.15.82:0/3497566204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:48.631 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:48 smithi082 bash[79733]: audit 2023-10-09T04:55:48.278915+0000 mon.a (mon.0) 1816 : audit 0 from='client.? 172.21.15.82:0/3497566204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:48.632 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:55:48.636 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.637 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:55:48.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:48.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:48 smithi153 bash[59565]: audit 2023-10-09T04:55:48.278915+0000 mon.a (mon.0) 1816 : audit 0 from='client.? 172.21.15.82:0/3497566204' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:48.771 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:48.794 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:48.898 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:48.928 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:48.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:49.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:49.056 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:49.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:49.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:49.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:49.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:49.326 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:49 smithi153 bash[59565]: cluster 2023-10-09T04:55:48.879124+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:55:49.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:49.465 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:49.570 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:49.577 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:49 smithi082 bash[77063]: cluster 2023-10-09T04:55:48.879124+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:55:49.577 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:49 smithi082 bash[79733]: cluster 2023-10-09T04:55:48.879124+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1377: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:55:49.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:49.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:49.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:49.828 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:49.837 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:49] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T04:55:49.851 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:49.954 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:49.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:49.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:49.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:50.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:50.217 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:50.255 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:50.349 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:50 smithi082 bash[77063]: audit 2023-10-09T04:55:50.134067+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:50.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:50.410 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:50.516 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:50.667 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:50 smithi153 bash[59565]: audit 2023-10-09T04:55:50.134067+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:50.702 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:50 smithi082 bash[79733]: audit 2023-10-09T04:55:50.134067+0000 mon.a (mon.0) 1817 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:50.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:50.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:50.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.972 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:50.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:51.078 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:51.110 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:51.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:51.243 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:51.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:51.377 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:51.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:51.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:51.523 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:51 smithi082 bash[77063]: cluster 2023-10-09T04:55:50.879572+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:55:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:51.525 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:51 smithi082 bash[79733]: cluster 2023-10-09T04:55:50.879572+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:55:51.625 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:51.650 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:51 smithi153 bash[59565]: cluster 2023-10-09T04:55:50.879572+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1378: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:55:51.756 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:51.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:51.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:51.934 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:51.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:52.039 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:52.066 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:52.173 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:52.198 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:52.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:52.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:52.386 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:55:52.389 INFO:teuthology.orchestra.run.smithi082.stderr:set pool 10 pgp_num to 16 2023-10-09T04:55:52.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:52.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:52.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:52.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:52 smithi153 bash[59565]: audit 2023-10-09T04:55:51.781213+0000 mon.a (mon.0) 1818 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:52 smithi153 bash[59565]: audit 2023-10-09T04:55:51.908969+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:55:52.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:52 smithi082 bash[77063]: audit 2023-10-09T04:55:51.781213+0000 mon.a (mon.0) 1818 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:52.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:52 smithi082 bash[77063]: audit 2023-10-09T04:55:51.908969+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:55:52.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:52 smithi082 bash[79733]: audit 2023-10-09T04:55:51.781213+0000 mon.a (mon.0) 1818 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:52.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:52 smithi082 bash[79733]: audit 2023-10-09T04:55:51.908969+0000 mon.a (mon.0) 1819 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:55:52.733 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:52.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:52.876 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:52.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:52.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.006 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:53.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:53.137 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:53.245 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:53.272 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.363994+0000 mon.a (mon.0) 1820 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.372140+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.372451+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: audit 2023-10-09T04:55:52.379382+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: audit 2023-10-09T04:55:52.379496+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.379654+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.383168+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.372 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:53 smithi153 bash[59565]: cluster 2023-10-09T04:55:52.880040+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T04:55:53.377 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:53.405 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:53.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:53.520 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.363994+0000 mon.a (mon.0) 1820 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.372140+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.372451+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: audit 2023-10-09T04:55:52.379382+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: audit 2023-10-09T04:55:52.379496+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.379654+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.383168+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[77063]: cluster 2023-10-09T04:55:52.880040+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T04:55:53.521 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.363994+0000 mon.a (mon.0) 1820 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.372140+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.372451+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: audit 2023-10-09T04:55:52.379382+0000 mon.a (mon.0) 1821 : audit 1 from='client.? 172.21.15.82:0/2366348037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: audit 2023-10-09T04:55:52.379496+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.82:0/262116169' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.379654+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.383168+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T04:55:53.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:53 smithi082 bash[79733]: cluster 2023-10-09T04:55:52.880040+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T04:55:53.543 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:53.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:53.649 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.672 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:53.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:53.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:53.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:53.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:53.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:54.036 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:54.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:54.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:55:54.163 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:54.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:54.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:54.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:54.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:54.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:54.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:54.838 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.877 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:54.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:54.985 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:55.020 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:55.130 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:55.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:55.259 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.292 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:55.396 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:55.527 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:55.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:55.667 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.695 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:55.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:55.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:55.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:55.973 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:55.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:56.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:56.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:56.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:56.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:56.233 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:56 smithi153 bash[59565]: cluster 2023-10-09T04:55:54.880553+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:55:56.259 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:56.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:56.417 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:56.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:56 smithi082 bash[77063]: cluster 2023-10-09T04:55:54.880553+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:55:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:56 smithi082 bash[79733]: cluster 2023-10-09T04:55:54.880553+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:55:56.521 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:56.556 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:56.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:56.683 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:56.787 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:56.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:56.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:56.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:56.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:57.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.102 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:57.120 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:55:57.120 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:55:57.120 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:135816 10.5 10:a4fb1a99:::benchmark_data_smithi082_223129_object30367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.091956+0000", 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.021572936000000001, 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.021614476000000001, 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:55:57.121 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 135816 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091956+0000", 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.122 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091956+0000", 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091954+0000", 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091961+0000", 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3029999999999996e-06 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091971+0000", 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0179e-05 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091983+0000", 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2187999999999999e-05 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.097262+0000", 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0052792619999999998 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.101336+0000", 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0040736000000000001 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.101797+0000", 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00046111200000000001 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:135825 10.e e309/293 10:7320df32:::benchmark_data_smithi082_223129_object30376:head v 309'8321, mlcod=309'8319)", 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.112539+0000", 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00099020199999999992, 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001117584, 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.127 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112539+0000", 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112539+0000", 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112538+0000", 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112546+0000", 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6060000000000003e-06 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112564+0000", 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7669999999999999e-05 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112572+0000", 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1669999999999999e-06 2023-10-09T04:55:57.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112615+0000", 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3069999999999999e-05 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.112636+0000", 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0332000000000001e-05 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.131 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:135826 10.3 10:c4d24fdb:::benchmark_data_smithi082_223129_object30377:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.113290+0000", 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.00023859799999999999, 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00042978100000000001, 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:55:57.132 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 135826 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113290+0000", 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113290+0000", 2023-10-09T04:55:57.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113289+0000", 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113296+0000", 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0870000000000001e-06 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113307+0000", 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1608e-05 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113316+0000", 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.7080000000000003e-06 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:55:57.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113470+0000", 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015370100000000001 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113644+0000", 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00017401 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.136 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:135827 10.3 10.4d9fe163 (undecoded) ondisk+write+known_if_redirected e309)", 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.113503+0000", 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 2.6211999999999999e-05, 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00028361899999999998, 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:55:57.137 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 135827 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113503+0000", 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.138 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113503+0000", 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113502+0000", 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113507+0000", 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.729e-06 2023-10-09T04:55:57.139 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113517+0000", 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0039e-05 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113524+0000", 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9739999999999996e-06 2023-10-09T04:55:57.140 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:135803 10.5 10:a8025d83:::benchmark_data_smithi082_223129_object30354:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.070867+0000", 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.042662314, 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.046259124999999998, 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 135803 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070867+0000", 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.142 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070867+0000", 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070865+0000", 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.143 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070873+0000", 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3440000000000001e-06 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070890+0000", 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7527999999999999e-05 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.144 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.070900+0000", 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2289999999999992e-06 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.071037+0000", 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000136947 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.071363+0000", 2023-10-09T04:55:57.145 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00032602999999999999 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.071751+0000", 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00038794900000000001 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091482+0000", 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019731180000000001 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.146 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.097169+0000", 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0056875880000000004 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:135805 10.5 10:ad16f7b1:::benchmark_data_smithi082_223129_object30356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:55:57.077588+0000", 2023-10-09T04:55:57.147 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.035940403000000003, 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0396472, 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 135805 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:55:57.148 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.077588+0000", 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.077588+0000", 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.077587+0000", 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:55:57.149 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.077595+0000", 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.0160000000000008e-06 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.085089+0000", 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0074947959999999997 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.150 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.085104+0000", 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4713e-05 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091510+0000", 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0064057259999999996 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:55:57.151 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091645+0000", 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013459799999999999 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.091775+0000", 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013012899999999999 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.111544+0000", 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.019769406 2023-10-09T04:55:57.152 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:55:57.113546+0000", 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0020016040000000001 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 6 2023-10-09T04:55:57.153 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:55:57.212 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:57.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:55:57.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:57.346 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:57.369 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:57.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:57.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:57.603 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:57.611 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:57 smithi153 bash[59565]: audit 2023-10-09T04:55:56.592787+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.611 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:57 smithi153 bash[59565]: audit 2023-10-09T04:55:56.601933+0000 mon.a (mon.0) 1825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.611 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:57 smithi153 bash[59565]: cluster 2023-10-09T04:55:56.881272+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T04:55:57.631 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:57.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.760 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:57.864 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[77063]: audit 2023-10-09T04:55:56.592787+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[77063]: audit 2023-10-09T04:55:56.601933+0000 mon.a (mon.0) 1825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[77063]: cluster 2023-10-09T04:55:56.881272+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[79733]: audit 2023-10-09T04:55:56.592787+0000 mon.a (mon.0) 1824 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[79733]: audit 2023-10-09T04:55:56.601933+0000 mon.a (mon.0) 1825 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:55:57.873 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:57 smithi082 bash[79733]: cluster 2023-10-09T04:55:56.881272+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-10-09T04:55:57.895 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:57.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:57.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:58.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:58.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:58.136 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:58.165 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:55:58.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:58.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:58.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:58.539 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.565 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:58.672 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:58.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:58.807 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:58.830 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:58.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:58.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:58.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:59.070 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:59.202 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:59.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:59.365 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:59.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:59.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:59.620 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:59.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:55:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:55:59] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.33.4" 2023-10-09T04:55:59.759 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:59.831 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:59.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:59.942 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:59.970 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:55:59.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:00.075 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:00.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:59 smithi082 bash[79733]: cluster 2023-10-09T04:55:58.881864+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T04:56:00.084 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:55:59 smithi082 bash[79733]: audit 2023-10-09T04:55:59.300300+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:00.085 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:59 smithi082 bash[77063]: cluster 2023-10-09T04:55:58.881864+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T04:56:00.085 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:55:59 smithi082 bash[77063]: audit 2023-10-09T04:55:59.300300+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:00.105 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:00.208 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:00.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:59 smithi153 bash[59565]: cluster 2023-10-09T04:55:58.881864+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T04:56:00.216 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:55:59 smithi153 bash[59565]: audit 2023-10-09T04:55:59.300300+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:00.219 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:56:00.219 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:00.220 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:56:00.220 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T04:56:00.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T04:56:00.232 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:00.338 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:00.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:00.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:00.629 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:00.681 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:00.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:00.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:00.876 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:00.905 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:00.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:01.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:01.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:01 smithi153 bash[59565]: audit 2023-10-09T04:56:00.100559+0000 mon.a (mon.0) 1827 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:01 smithi153 bash[59565]: audit 2023-10-09T04:56:00.115230+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:01 smithi153 bash[59565]: audit 2023-10-09T04:56:00.755260+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:01 smithi153 bash[59565]: audit 2023-10-09T04:56:00.759907+0000 mon.a (mon.0) 1830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:01.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:01 smithi153 bash[59565]: audit 2023-10-09T04:56:00.768610+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.177 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:01.285 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:01.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:01.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.435 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[77063]: audit 2023-10-09T04:56:00.100559+0000 mon.a (mon.0) 1827 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[77063]: audit 2023-10-09T04:56:00.115230+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[77063]: audit 2023-10-09T04:56:00.755260+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[77063]: audit 2023-10-09T04:56:00.759907+0000 mon.a (mon.0) 1830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[77063]: audit 2023-10-09T04:56:00.768610+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[79733]: audit 2023-10-09T04:56:00.100559+0000 mon.a (mon.0) 1827 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[79733]: audit 2023-10-09T04:56:00.115230+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[79733]: audit 2023-10-09T04:56:00.755260+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:56:01.436 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[79733]: audit 2023-10-09T04:56:00.759907+0000 mon.a (mon.0) 1830 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:56:01.437 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:01 smithi082 bash[79733]: audit 2023-10-09T04:56:00.768610+0000 mon.a (mon.0) 1831 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:01.464 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:01.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:01.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:01.710 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:01.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:01.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:01.869 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:01.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:01.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:01.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:02.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.128 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:02.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:02.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:02.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:02.376 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:02 smithi082 bash[77063]: cluster 2023-10-09T04:56:00.882368+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T04:56:02.376 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:02 smithi082 bash[79733]: cluster 2023-10-09T04:56:00.882368+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T04:56:02.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:02 smithi153 bash[59565]: cluster 2023-10-09T04:56:00.882368+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1384: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 712 op/s 2023-10-09T04:56:02.509 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:02.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:02.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:02.722 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:02.725 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:02.796 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.838 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:02.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:02.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:02.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:03.074 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:03.117 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:03.224 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:03.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:03.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:03.386 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:03.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:03.493 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:03.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:03.628 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:03.666 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:03.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:03.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:03.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:03.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:03.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:04.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:04.068 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:04.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:04.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:04.196 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:04.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:04.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:04 smithi082 bash[77063]: cluster 2023-10-09T04:56:02.882955+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:56:04.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:04 smithi082 bash[79733]: cluster 2023-10-09T04:56:02.882955+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:56:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:04 smithi153 bash[59565]: cluster 2023-10-09T04:56:02.882955+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1385: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:56:04.444 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:04.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:04.571 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:04.599 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:04.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:04.762 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:04.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:04.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:05.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:05.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:05.178 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:05.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:05.318 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:05.425 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:05.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:05.559 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:05.593 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:05.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:05.832 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:05.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:05.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:05.996 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:06.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:06.103 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:06.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:06.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:06.269 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:06.373 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:06.382 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:06 smithi153 bash[59565]: cluster 2023-10-09T04:56:04.883482+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:06.400 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:06.494 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:06 smithi082 bash[77063]: cluster 2023-10-09T04:56:04.883482+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:06.494 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:06 smithi082 bash[79733]: cluster 2023-10-09T04:56:04.883482+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1386: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:06.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:06.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:06.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:06.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:06.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:06.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:06.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:06.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:06.923 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:06.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:06.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:07.063 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.093 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:07.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:07.194 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:07.228 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:07.336 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:07.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:07.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:07.473 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:07.507 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:07.611 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:07.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:07.741 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:07.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:07.886 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:07.926 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:07.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:07.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:08.038 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.069 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:08.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:08.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:08 smithi082 bash[77063]: cluster 2023-10-09T04:56:06.884060+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:08.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:08 smithi082 bash[79733]: cluster 2023-10-09T04:56:06.884060+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:08.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:08.344 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:08.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:08 smithi153 bash[59565]: cluster 2023-10-09T04:56:06.884060+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1387: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:08.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.473 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:08.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:08.634 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:08.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:08.764 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:08.870 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:08.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:08.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:08.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:09.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:09.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:09.155 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:09.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:09.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:09.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:09.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:09.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:09.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:09.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:09.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:09.710 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:09.759 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:09] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.33.4" 2023-10-09T04:56:09.814 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:09.857 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:09.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:09.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:09.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:10.089 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:10.123 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:10.173 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:10 smithi082 bash[77063]: cluster 2023-10-09T04:56:08.884589+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T04:56:10.229 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:10.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:10 smithi153 bash[59565]: cluster 2023-10-09T04:56:08.884589+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T04:56:10.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:10.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:10.404 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:10.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:10 smithi082 bash[79733]: cluster 2023-10-09T04:56:08.884589+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1388: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-09T04:56:10.508 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:10.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:10.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:10.675 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:10.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:10.812 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:10.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:10.950 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:10.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:11.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:11.086 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:11.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:11.188 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:11.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:11.331 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:11.362 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:11.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:11.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:11.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:11.640 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:11.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:11.748 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:11.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:11.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:11.916 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:11.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:12.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:12.048 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:12.155 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:12.188 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:12.291 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:12.320 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:12 smithi153 bash[59565]: cluster 2023-10-09T04:56:10.886264+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:56:12.430 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.439 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:12 smithi082 bash[77063]: cluster 2023-10-09T04:56:10.886264+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:56:12.439 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:12 smithi082 bash[79733]: cluster 2023-10-09T04:56:10.886264+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:56:12.460 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:12.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:12.598 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:12.704 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.726 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:12.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:12.851 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:12.954 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.979 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:12.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:13.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:13.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:13.217 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:13.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:13.368 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:13.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:13.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:13.506 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:13.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:13.636 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:13.666 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:13.770 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:13.797 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:13.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:13.928 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:13.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:13.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:14.031 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:14.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:14.177 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:14.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:14.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:14.303 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:14.311 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:14 smithi082 bash[79733]: cluster 2023-10-09T04:56:12.886907+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:14.311 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:14 smithi082 bash[77063]: cluster 2023-10-09T04:56:12.886907+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:14.335 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:14.440 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:14.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:14.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:14.616 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:14.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:14 smithi153 bash[59565]: cluster 2023-10-09T04:56:12.886907+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:14.722 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:14.751 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:14.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:14.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:14.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:14.987 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.019 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:15.123 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:15.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:15.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.280 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:15.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.393 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:15 smithi082 bash[77063]: audit 2023-10-09T04:56:14.300302+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:15.393 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:15 smithi082 bash[79733]: audit 2023-10-09T04:56:14.300302+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:15.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:15.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:15.546 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:15.651 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:15.661 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:15 smithi153 bash[59565]: audit 2023-10-09T04:56:14.300302+0000 mon.a (mon.0) 1832 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:15.675 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:15.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:15.811 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:15.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:15.914 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:15.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:15.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:16.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:16.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:16.177 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:16.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:16.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:16.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:16 smithi082 bash[77063]: cluster 2023-10-09T04:56:14.887376+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T04:56:16.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:16 smithi082 bash[79733]: cluster 2023-10-09T04:56:14.887376+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T04:56:16.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:16.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:16.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:16.446 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:16 smithi153 bash[59565]: cluster 2023-10-09T04:56:14.887376+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T04:56:16.473 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:16.576 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:16.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:16.640 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:56:16.676 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:56:16.707 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:16.726 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:56:16.743 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:16.763 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:56:16.797 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:56:16.841 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:56:16.847 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:16.883 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:56:16.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:16.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:16.918 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.931 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:56:16.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:17.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:17.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:17.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:17.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:17.275 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:17.306 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:17.412 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:17.419 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:17 smithi082 bash[77063]: audit 2023-10-09T04:56:16.598064+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.82:0/493156489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:17.419 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:17 smithi082 bash[79733]: audit 2023-10-09T04:56:16.598064+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.82:0/493156489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:17.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:17.543 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:17.551 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:17 smithi153 bash[59565]: audit 2023-10-09T04:56:16.598064+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.82:0/493156489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:17.571 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:17.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:17.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:17.705 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:17.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:17.840 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:17.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:17.982 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:17.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:17.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:18.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:18.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:18.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:18.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:18 smithi082 bash[77063]: cluster 2023-10-09T04:56:16.887976+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:56:18.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:18 smithi082 bash[79733]: cluster 2023-10-09T04:56:16.887976+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:56:18.254 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:18.358 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:18.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:18.500 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:18.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:18.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:18.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:18 smithi153 bash[59565]: cluster 2023-10-09T04:56:16.887976+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1392: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:56:18.670 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:18.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:18.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:18.917 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:18.948 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:18.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:19.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:19.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:19.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:19.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:19.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:19.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:19.464 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:19.498 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:19.606 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:19.633 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:19.702 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:19] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T04:56:19.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:19.780 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:19.883 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:19.907 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:19.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:20.011 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:20.055 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:20.160 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:20.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:20.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:20.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:20.294 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:20 smithi082 bash[77063]: cluster 2023-10-09T04:56:18.888573+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:56:20.294 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:20 smithi082 bash[79733]: cluster 2023-10-09T04:56:18.888573+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:56:20.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:20 smithi153 bash[59565]: cluster 2023-10-09T04:56:18.888573+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:56:20.425 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:56:20.425 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:56:20.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:20.461 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:20.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.590 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:20.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.718 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:20.829 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:20.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:20.965 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:21.002 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:21.107 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:21.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:21.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:21.258 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:21.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:21.395 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:21.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:21.533 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:21.638 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:21.668 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:21.774 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:21.820 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:21.923 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:21.946 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:21.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:22.049 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:22.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:22.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:22.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:56:22.221 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:22.227 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:56:22.231 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T04:56:22.326 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:22.335 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:56:22.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:22.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:22.461 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:22 smithi153 bash[59565]: cluster 2023-10-09T04:56:20.889134+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T04:56:22.461 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:22 smithi153 bash[59565]: audit 2023-10-09T04:56:21.396038+0000 mon.a (mon.0) 1833 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:22.462 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:22 smithi153 bash[59565]: audit 2023-10-09T04:56:21.643805+0000 mon.a (mon.0) 1834 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:56:22.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[77063]: cluster 2023-10-09T04:56:20.889134+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T04:56:22.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[77063]: audit 2023-10-09T04:56:21.396038+0000 mon.a (mon.0) 1833 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:22.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[77063]: audit 2023-10-09T04:56:21.643805+0000 mon.a (mon.0) 1834 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:56:22.484 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:22.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[79733]: cluster 2023-10-09T04:56:20.889134+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1394: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T04:56:22.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[79733]: audit 2023-10-09T04:56:21.396038+0000 mon.a (mon.0) 1833 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:22.484 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:22 smithi082 bash[79733]: audit 2023-10-09T04:56:21.643805+0000 mon.a (mon.0) 1834 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:56:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:22.590 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:22.619 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:22.724 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:22.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:22.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:22.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:22.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:22.997 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:23.021 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:23.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:23.260 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:23.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:23.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:23.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:23.422 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:23.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: cluster 2023-10-09T04:56:22.209039+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: cluster 2023-10-09T04:56:22.219027+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: audit 2023-10-09T04:56:22.221276+0000 mon.a (mon.0) 1836 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: audit 2023-10-09T04:56:22.221361+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: cluster 2023-10-09T04:56:22.221424+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[77063]: cluster 2023-10-09T04:56:22.226386+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: cluster 2023-10-09T04:56:22.209039+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: cluster 2023-10-09T04:56:22.219027+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: audit 2023-10-09T04:56:22.221276+0000 mon.a (mon.0) 1836 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: audit 2023-10-09T04:56:22.221361+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:56:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: cluster 2023-10-09T04:56:22.221424+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T04:56:23.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:23 smithi082 bash[79733]: cluster 2023-10-09T04:56:22.226386+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:23.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: cluster 2023-10-09T04:56:22.209039+0000 mon.a (mon.0) 1835 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: cluster 2023-10-09T04:56:22.219027+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: audit 2023-10-09T04:56:22.221276+0000 mon.a (mon.0) 1836 : audit 1 from='client.? 172.21.15.82:0/2053685624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: audit 2023-10-09T04:56:22.221361+0000 mon.a (mon.0) 1837 : audit 1 from='client.? 172.21.15.82:0/3318017030' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: cluster 2023-10-09T04:56:22.221424+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e310: 8 total, 8 up, 7 in 2023-10-09T04:56:23.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:23 smithi153 bash[59565]: cluster 2023-10-09T04:56:22.226386+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T04:56:23.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:23.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.828 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:23.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:23.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:23.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:24.067 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:24.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:24.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:24.236 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:24.341 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:24.350 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:24 smithi082 bash[77063]: cluster 2023-10-09T04:56:22.889670+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:24.350 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:24 smithi082 bash[79733]: cluster 2023-10-09T04:56:22.889670+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:24.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:24.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:24.507 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:24.610 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:24.644 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:24.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:24.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:24 smithi153 bash[59565]: cluster 2023-10-09T04:56:22.889670+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1396: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T04:56:24.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:24.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:24.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:24.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:24.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:25.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:25.051 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:25.154 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:25.182 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:25.289 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:25.317 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:25.419 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:25.453 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:25.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:25.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:25.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:25.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:25.717 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:25.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:25.857 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:25.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:56:25.964 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:25.988 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:25.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:26.091 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:26.114 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:26.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.221 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:26 smithi082 bash[77063]: cluster 2023-10-09T04:56:24.890187+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:56:26.227 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:26 smithi153 bash[59565]: cluster 2023-10-09T04:56:24.890187+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:56:26.242 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:26.346 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:26.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:26.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:26 smithi082 bash[79733]: cluster 2023-10-09T04:56:24.890187+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1397: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:56:26.485 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:26.510 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:26.613 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:26.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:26.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:26.745 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:26.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:26.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:26.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:27.042 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.073 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:27.073 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:27.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:27.181 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:27.322 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:27.345 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:27.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:27.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.457 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:27 smithi153 bash[59565]: cluster 2023-10-09T04:56:26.890773+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T04:56:27.478 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:27.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:27 smithi082 bash[77063]: cluster 2023-10-09T04:56:26.890773+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T04:56:27.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:27 smithi082 bash[79733]: cluster 2023-10-09T04:56:26.890773+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v1398: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T04:56:27.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:27.615 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:27.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:27.746 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:27.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:27.850 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:27.880 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:27.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:27.987 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:28.024 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:28.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:28.157 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:28.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:28.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:28.399 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:28.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:28.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:28.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:28.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:28.936 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:28.943 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:56:28.943 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:28.943 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T04:56:28.943 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T04:56:28.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:28.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:29.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:29.098 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:29.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:29.205 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:29.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:29.365 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:29.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:29.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:29.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:29.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:29.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:29.659 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:29.765 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:29.802 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:29.906 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:29.915 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:29] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T04:56:29.936 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:29.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:29.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:30.044 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:30.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:30.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:30.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:29 smithi153 bash[59565]: cluster 2023-10-09T04:56:28.891481+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T04:56:30.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:29 smithi153 bash[59565]: audit 2023-10-09T04:56:29.304043+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:30.185 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:30.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:29 smithi082 bash[77063]: cluster 2023-10-09T04:56:28.891481+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T04:56:30.194 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:29 smithi082 bash[77063]: audit 2023-10-09T04:56:29.304043+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:30.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:29 smithi082 bash[79733]: cluster 2023-10-09T04:56:28.891481+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T04:56:30.195 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:29 smithi082 bash[79733]: audit 2023-10-09T04:56:29.304043+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:30.211 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:30.318 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:30.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:30.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:30.581 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.614 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:30.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.744 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:30.847 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:30.879 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:30.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:30.984 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:31.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:31.104 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:31.131 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:31.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:31.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.394 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:31.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:31.519 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:31.622 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:31.761 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:31.814 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:31.899 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.926 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:31.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:32.029 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:32.063 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:32.166 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:32 smithi082 bash[77063]: cluster 2023-10-09T04:56:30.891986+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T04:56:32.167 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:32.198 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:32.301 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:32.328 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:32 smithi153 bash[59565]: cluster 2023-10-09T04:56:30.891986+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T04:56:32.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:32.444 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:32 smithi082 bash[79733]: cluster 2023-10-09T04:56:30.891986+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T04:56:32.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:32.562 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:32.594 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:32.700 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:32.736 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:32.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:32.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:32.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:32.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:32.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:33.000 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:33.105 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.141 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:33.245 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:33.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:33.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:33.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:33.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:33.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:33.671 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:33.775 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:33.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:33.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:33.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.943 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:33.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:34.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:34.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:34.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:34.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:34 smithi153 bash[59565]: cluster 2023-10-09T04:56:32.892933+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T04:56:34.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:34.221 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:34.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:34.315 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:34 smithi082 bash[77063]: cluster 2023-10-09T04:56:32.892933+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T04:56:34.315 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:34 smithi082 bash[79733]: cluster 2023-10-09T04:56:32.892933+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 759 op/s 2023-10-09T04:56:34.326 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:34.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:34.373 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:56:34.373 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:56:34.396 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:34.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:34.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:34.548 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:56:34.595 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:56:34.731 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:34.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:56:34.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:56:34.732 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:56:34.756 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:56:34.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:34.797 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:56:34.834 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:56:34.840 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.840 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.840 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.841 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.841 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.841 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.841 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.841 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:56:34.867 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:34.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:34.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:35.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.036 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:35.141 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:35.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:35.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:35.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.416 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:35 smithi082 bash[77063]: audit 2023-10-09T04:56:34.496514+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.82:0/2260035658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:35.417 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:35 smithi082 bash[79733]: audit 2023-10-09T04:56:34.496514+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.82:0/2260035658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:35.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:35.548 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:35.556 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:35 smithi153 bash[59565]: audit 2023-10-09T04:56:34.496514+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.82:0/2260035658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:35.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:35.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:35.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:35.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:35.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:35.841 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:35.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:35.944 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:35.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:35.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:36.078 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.110 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:36.214 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:36.231 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:36.250 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:36 smithi082 bash[79733]: cluster 2023-10-09T04:56:34.893446+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:36.250 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:36 smithi082 bash[77063]: cluster 2023-10-09T04:56:34.893446+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:36.334 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:36.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:36.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:36.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:36.485 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:36.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:36.589 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:36 smithi153 bash[59565]: cluster 2023-10-09T04:56:34.893446+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T04:56:36.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:36.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:36.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.896 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:36.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:36.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:36.998 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:37.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:37.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:37.161 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:37.265 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:37.291 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:37.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:37.393 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:37.526 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.561 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:37.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:37.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:37.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:37.836 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:56:37.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:37.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:37.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:38.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:38.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:38.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:38 smithi082 bash[77063]: cluster 2023-10-09T04:56:36.897035+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:38.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:38 smithi082 bash[79733]: cluster 2023-10-09T04:56:36.897035+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:38.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:38.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:38.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:38.399 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:38.504 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:38.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:38.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:38.633 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:38 smithi153 bash[59565]: cluster 2023-10-09T04:56:36.897035+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:56:38.662 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:38.767 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:38.803 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:38.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:38.935 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:38.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:39.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.067 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:39.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:39.200 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:39.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:39.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:39.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:39.468 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:39.573 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:39.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:39.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:39.756 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:39.858 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:56:39.863 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:39.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:39] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T04:56:39.989 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:40.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:40.114 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:40.145 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:40.249 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:40.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:40.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:40.411 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:40.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:40 smithi082 bash[77063]: cluster 2023-10-09T04:56:38.897750+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:56:40.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:40 smithi082 bash[79733]: cluster 2023-10-09T04:56:38.897750+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:56:40.516 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:40.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:40 smithi153 bash[59565]: cluster 2023-10-09T04:56:38.897750+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:56:40.546 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:40.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:40.684 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:40.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:40.824 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:40.928 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:40.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:40.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:41.060 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:41.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.219 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:41.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:41.365 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:41.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:41.499 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:41.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:41.635 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:41.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:41.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:41.900 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:41.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:41.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:42.038 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:42.075 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:42.179 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:42.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:42.329 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:42.375 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:42.430 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:56:42.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:42.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:42 smithi082 bash[77063]: cluster 2023-10-09T04:56:40.898256+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:56:42.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:42 smithi082 bash[77063]: audit 2023-10-09T04:56:41.239782+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:42.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:42 smithi082 bash[79733]: cluster 2023-10-09T04:56:40.898256+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:56:42.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:42 smithi082 bash[79733]: audit 2023-10-09T04:56:41.239782+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:42.508 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:42.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:42.645 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:42 smithi153 bash[59565]: cluster 2023-10-09T04:56:40.898256+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:56:42.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:42 smithi153 bash[59565]: audit 2023-10-09T04:56:41.239782+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:42.748 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:42.783 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:42.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:42.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:42.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:43.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.076 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:43.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:43.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:43.203 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:43.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:43.340 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:43.443 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.196207+0000 mon.a (mon.0) 1841 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.196246+0000 mon.a (mon.0) 1842 : cluster 1 Cluster is now healthy 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.200184+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: audit 2023-10-09T04:56:42.203048+0000 mon.a (mon.0) 1843 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.204229+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.208834+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.459 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.196207+0000 mon.a (mon.0) 1841 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:43.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.196246+0000 mon.a (mon.0) 1842 : cluster 1 Cluster is now healthy 2023-10-09T04:56:43.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.200184+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: audit 2023-10-09T04:56:42.203048+0000 mon.a (mon.0) 1843 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:43.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.204229+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T04:56:43.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:43 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.208834+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:43.594 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:43.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.196207+0000 mon.a (mon.0) 1841 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:43.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.196246+0000 mon.a (mon.0) 1842 : cluster 1 Cluster is now healthy 2023-10-09T04:56:43.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.200184+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.607 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: audit 2023-10-09T04:56:42.203048+0000 mon.a (mon.0) 1843 : audit 1 from='client.? 172.21.15.82:0/3074343982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:43.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.204229+0000 mon.a (mon.0) 1844 : cluster 0 osdmap e311: 8 total, 8 up, 7 in 2023-10-09T04:56:43.608 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:43 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.208834+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T04:56:43.623 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:43.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:43.754 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:43.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:43.901 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:43.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:43.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:44.005 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:44.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:44.170 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:44.276 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:44.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:44.407 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:44 smithi082 bash[77063]: cluster 2023-10-09T04:56:42.898802+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T04:56:44.424 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:44 smithi082 bash[79733]: cluster 2023-10-09T04:56:42.898802+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T04:56:44.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:44 smithi153 bash[59565]: cluster 2023-10-09T04:56:42.898802+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T04:56:44.441 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:44.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:44.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:44.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:44.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:44 smithi082 bash[77063]: debug 2023-10-09T04:56:44.641+0000 7f3fc98c6700 -1 mon.a@0(leader).osd e311 definitely_dead 0 2023-10-09T04:56:44.728 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:44.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.862 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:44.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:44.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:44.999 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:45.108 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:45.133 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:45.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:45.273 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:45.324 INFO:teuthology.orchestra.run.smithi082.stderr:marked in osd.4. 2023-10-09T04:56:45.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:45.380 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[77063]: audit 2023-10-09T04:56:44.304919+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:45.380 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[77063]: audit 2023-10-09T04:56:44.306107+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:45.380 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[77063]: audit 2023-10-09T04:56:44.647337+0000 mon.a (mon.0) 1847 : audit 0 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:56:45.380 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[77063]: cluster 2023-10-09T04:56:44.899331+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T04:56:45.380 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[79733]: audit 2023-10-09T04:56:44.304919+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:45.381 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[79733]: audit 2023-10-09T04:56:44.306107+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:45.381 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[79733]: audit 2023-10-09T04:56:44.647337+0000 mon.a (mon.0) 1847 : audit 0 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:56:45.381 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:45 smithi082 bash[79733]: cluster 2023-10-09T04:56:44.899331+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T04:56:45.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:45.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:45.525 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:45 smithi153 bash[59565]: audit 2023-10-09T04:56:44.304919+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:56:45.525 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:45 smithi153 bash[59565]: audit 2023-10-09T04:56:44.306107+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:45.525 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:45 smithi153 bash[59565]: audit 2023-10-09T04:56:44.647337+0000 mon.a (mon.0) 1847 : audit 0 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:56:45.525 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:45 smithi153 bash[59565]: cluster 2023-10-09T04:56:44.899331+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T04:56:45.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:45.647 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:45.679 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:45.751 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:45.752 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:45.782 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:45.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:45.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:45.944 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:45.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:46.047 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:46.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:56:46.074 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:46.176 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:46.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:46.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:46.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:46.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:46.577 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:46.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:46 smithi153 bash[59565]: cluster 2023-10-09T04:56:45.312502+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:46 smithi153 bash[59565]: audit 2023-10-09T04:56:45.319535+0000 mon.a (mon.0) 1848 : audit 1 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:56:46.587 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:46 smithi153 bash[59565]: cluster 2023-10-09T04:56:45.319594+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:56:46.588 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:46 smithi153 bash[59565]: cluster 2023-10-09T04:56:45.320032+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:46.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:46.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.715 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[79733]: cluster 2023-10-09T04:56:45.312502+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[79733]: audit 2023-10-09T04:56:45.319535+0000 mon.a (mon.0) 1848 : audit 1 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[79733]: cluster 2023-10-09T04:56:45.319594+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[79733]: cluster 2023-10-09T04:56:45.320032+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[77063]: cluster 2023-10-09T04:56:45.312502+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[77063]: audit 2023-10-09T04:56:45.319535+0000 mon.a (mon.0) 1848 : audit 1 from='client.? 172.21.15.82:0/2044854832' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[77063]: cluster 2023-10-09T04:56:45.319594+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:56:46.716 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:46 smithi082 bash[77063]: cluster 2023-10-09T04:56:45.320032+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T04:56:46.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:46.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:46.967 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:46.995 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:46.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:47.098 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:47.130 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:47.234 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:47.266 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:47.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:47.370 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:47.395 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:47.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:47.522 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:47.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.322423+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.326876+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.327208+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:56:47.624 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.327739+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.329856+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.331167+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.332010+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.332938+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.333507+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.335193+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.335835+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.338216+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.340254+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.342302+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[77063]: cluster 2023-10-09T04:56:46.900132+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 active+recovering, 2 remapped+peering, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1/129162 objects misplaced (0.001%); 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.322423+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.326876+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.327208+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.327739+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.329856+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.331167+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.332010+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.332938+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.626 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.333507+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.335193+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.335835+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.338216+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.340254+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.342302+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:47 smithi082 bash[79733]: cluster 2023-10-09T04:56:46.900132+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 active+recovering, 2 remapped+peering, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1/129162 objects misplaced (0.001%); 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:56:47.628 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T04:56:47.628 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:47.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.322423+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.326876+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.327208+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:56:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.327739+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.329856+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.331167+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.332010+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.332938+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.333507+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.335193+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.335835+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.338216+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.340254+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.342302+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T04:56:47.677 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:47 smithi153 bash[59565]: cluster 2023-10-09T04:56:46.900132+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v1411: 89 pgs: 1 active+recovering, 2 remapped+peering, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1/129162 objects misplaced (0.001%); 9 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:56:47.755 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:47.779 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:47.883 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:47.915 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:47.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:48.019 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.059 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:48.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:48.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:48.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:48.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:48.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.474 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:48.583 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.330156+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.333880+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.333986+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.334319+0000 osd.2 (osd.2) 2 : cluster 0 10.e starting backfill to osd.4 from (0'0,0'0] MAX to 311'10335 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.334528+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:56:48.594 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.334746+0000 osd.0 (osd.0) 5 : cluster 0 10.2 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10543 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.336935+0000 osd.0 (osd.0) 6 : cluster 0 10.1 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10728 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.337076+0000 osd.5 (osd.5) 228 : cluster 0 10.6 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10592 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.337375+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.337794+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.338981+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.339977+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.342495+0000 osd.6 (osd.6) 187 : cluster 0 10.0 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10544 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.342906+0000 osd.6 (osd.6) 188 : cluster 0 10.d starting backfill to osd.4 from (0'0,0'0] MAX to 311'10427 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.345899+0000 osd.0 (osd.0) 7 : cluster 0 10.4 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10614 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.349580+0000 osd.7 (osd.7) 268 : cluster 0 10.c starting backfill to osd.4 from (0'0,0'0] MAX to 311'10498 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.510197+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.595 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:48 smithi153 bash[59565]: cluster 2023-10-09T04:56:47.512876+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:48.711 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:48.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.330156+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.333880+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.333986+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.721 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.334319+0000 osd.2 (osd.2) 2 : cluster 0 10.e starting backfill to osd.4 from (0'0,0'0] MAX to 311'10335 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.334528+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.334746+0000 osd.0 (osd.0) 5 : cluster 0 10.2 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10543 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.336935+0000 osd.0 (osd.0) 6 : cluster 0 10.1 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10728 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.337076+0000 osd.5 (osd.5) 228 : cluster 0 10.6 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10592 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.337375+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.337794+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.338981+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.339977+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.342495+0000 osd.6 (osd.6) 187 : cluster 0 10.0 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10544 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.342906+0000 osd.6 (osd.6) 188 : cluster 0 10.d starting backfill to osd.4 from (0'0,0'0] MAX to 311'10427 2023-10-09T04:56:48.722 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.345899+0000 osd.0 (osd.0) 7 : cluster 0 10.4 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10614 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.349580+0000 osd.7 (osd.7) 268 : cluster 0 10.c starting backfill to osd.4 from (0'0,0'0] MAX to 311'10498 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.510197+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[77063]: cluster 2023-10-09T04:56:47.512876+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.330156+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.333880+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.333986+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.334319+0000 osd.2 (osd.2) 2 : cluster 0 10.e starting backfill to osd.4 from (0'0,0'0] MAX to 311'10335 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.334528+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.334746+0000 osd.0 (osd.0) 5 : cluster 0 10.2 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10543 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.336935+0000 osd.0 (osd.0) 6 : cluster 0 10.1 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10728 2023-10-09T04:56:48.723 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.337076+0000 osd.5 (osd.5) 228 : cluster 0 10.6 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10592 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.337375+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.337794+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.338981+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.339977+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.342495+0000 osd.6 (osd.6) 187 : cluster 0 10.0 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10544 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.342906+0000 osd.6 (osd.6) 188 : cluster 0 10.d starting backfill to osd.4 from (0'0,0'0] MAX to 311'10427 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.345899+0000 osd.0 (osd.0) 7 : cluster 0 10.4 starting backfill to osd.4 from (0'0,0'0] MAX to 311'10614 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.349580+0000 osd.7 (osd.7) 268 : cluster 0 10.c starting backfill to osd.4 from (0'0,0'0] MAX to 311'10498 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.510197+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.724 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:48 smithi082 bash[79733]: cluster 2023-10-09T04:56:47.512876+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T04:56:48.738 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:48.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:48.844 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.876 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:48.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:48.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:49.030 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:49.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:49.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:49.284 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:49.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.347472+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T04:56:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.349560+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.350087+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.351793+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.352439+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.356560+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:49 smithi153 bash[59565]: cluster 2023-10-09T04:56:48.900721+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v1414: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 80 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 1/124815 objects degraded (0.001%); 15478/124815 objects misplaced (12.401%); 176 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:49.428 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:49.459 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:49.564 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:49.607 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:49.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.347472+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T04:56:49.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.349560+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.350087+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.351793+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.352439+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.356560+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[79733]: cluster 2023-10-09T04:56:48.900721+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v1414: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 80 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 1/124815 objects degraded (0.001%); 15478/124815 objects misplaced (12.401%); 176 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.347472+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.349560+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.350087+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.351793+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.352439+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.356560+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T04:56:49.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[77063]: cluster 2023-10-09T04:56:48.900721+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v1414: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 80 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 1/124815 objects degraded (0.001%); 15478/124815 objects misplaced (12.401%); 176 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:49.701 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:49] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.33.4" 2023-10-09T04:56:49.713 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:49.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:49.864 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:49.887 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:49.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:49.990 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:50.016 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:50.122 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:50.156 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:50.261 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:50.287 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:50.356 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.344151+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Degraded data redundancy: 1/124815 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:56:50.356 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.356710+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.356 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.357631+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T04:56:50.356 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.362288+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.356 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.362728+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.364971+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.357 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[77063]: cluster 2023-10-09T04:56:49.367107+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.391 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:50.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.344151+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Degraded data redundancy: 1/124815 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:56:50.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.356710+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.357631+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T04:56:50.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.362288+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.362728+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.403 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.364971+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.403 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:50 smithi153 bash[59565]: cluster 2023-10-09T04:56:49.367107+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:50.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:50.555 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:50.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:50.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.344151+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Degraded data redundancy: 1/124815 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:56:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.356710+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.357631+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-10-09T04:56:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.362288+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.362728+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.364971+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:50 smithi082 bash[79733]: cluster 2023-10-09T04:56:49.367107+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T04:56:50.795 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:50.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:50.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:50.971 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:50.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:51.079 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:51.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:51.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:51.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:51.359 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:51.366 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:51 smithi082 bash[77063]: cluster 2023-10-09T04:56:50.901298+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v1416: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 79 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 1/124182 objects degraded (0.001%); 17982/124182 objects misplaced (14.480%); 154 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:51.366 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:51 smithi153 bash[59565]: cluster 2023-10-09T04:56:50.901298+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v1416: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 79 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 1/124182 objects degraded (0.001%); 17982/124182 objects misplaced (14.480%); 154 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:51.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:51.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:51.497 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:51.530 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:51.635 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:51.663 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:51.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:51 smithi082 bash[79733]: cluster 2023-10-09T04:56:50.901298+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v1416: 89 pgs: 1 activating+degraded+remapped, 1 active+remapped+backfilling, 3 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovering, 2 remapped+peering, 79 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 1/124182 objects degraded (0.001%); 17982/124182 objects misplaced (14.480%); 154 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T04:56:51.766 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:51.796 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:51.904 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:51.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:51.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:52.034 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:52.058 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:52.163 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:52.193 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:52.296 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:52.337 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:52.441 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.475 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:52.579 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.617 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:52.629 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:56:52.629 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:52.629 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:56:52.629 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.075504 2023-10-09T04:56:52.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.07550390487827374 2023-10-09T04:56:52.725 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.748 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:52.854 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:52.906 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:52.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:53.000 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:172440 10.5 e316/293 10:aa16bc73:::benchmark_data_smithi082_223129_object66991:head v 316'10853, mlcod=316'10851)", 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.994922+0000", 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0033556860000000001, 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0033858719999999998, 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.001 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994922+0000", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994922+0000", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994921+0000", 2023-10-09T04:56:53.002 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994929+0000", 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1100000000000003e-06 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994940+0000", 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1535000000000001e-05 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.003 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994948+0000", 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.7109999999999995e-06 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995028+0000", 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9827999999999995e-05 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.004 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995052+0000", 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3839e-05 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:172432 10.2 e316/313)", 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.995670+0000", 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.002607257, 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.002702458, 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.005 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995670+0000", 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995670+0000", 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.006 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995669+0000", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995675+0000", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2789999999999998e-06 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995685+0000", 2023-10-09T04:56:53.007 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.08e-05 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995692+0000", 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6499999999999999e-06 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.008 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:172434 10.2 e316/313)", 2023-10-09T04:56:53.009 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.995702+0000", 2023-10-09T04:56:53.009 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.002575057, 2023-10-09T04:56:53.009 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0027160880000000002, 2023-10-09T04:56:53.009 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.009 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995702+0000", 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.010 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995702+0000", 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995702+0000", 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995704+0000", 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2989999999999999e-06 2023-10-09T04:56:53.011 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995709+0000", 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1730000000000003e-06 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995714+0000", 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2969999999999997e-06 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.012 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:172434 10.2 e316/313)", 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.996006+0000", 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0022712489999999999, 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0024589709999999999, 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.013 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996006+0000", 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996006+0000", 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996005+0000", 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:53.014 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996010+0000", 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7339999999999999e-06 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996021+0000", 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0939e-05 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.015 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996029+0000", 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5909999999999998e-06 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:172432 10.2 10:4e22a7e2:::benchmark_data_smithi082_223129_object66983:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.974804+0000", 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.023473826999999999, 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023713998, 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.016 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 172432 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.974804+0000", 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.017 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.974804+0000", 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.974802+0000", 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.018 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.974809+0000", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.8360000000000003e-06 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.974822+0000", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.326e-05 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.975193+0000", 2023-10-09T04:56:53.019 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00037144899999999999 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.976732+0000", 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001538106 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.976889+0000", 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000157727 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977106+0000", 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000216918 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:172434 10.2 10:47be70c3:::benchmark_data_smithi082_223129_object66985:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.977571+0000", 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.020706634000000002, 2023-10-09T04:56:53.021 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.021024215999999998, 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 172434 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.022 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977571+0000", 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977571+0000", 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977569+0000", 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:53.023 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977575+0000", 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977586+0000", 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1538e-05 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.024 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.977594+0000", 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.571e-06 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985264+0000", 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0076704909999999998 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985427+0000", 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000162678 2023-10-09T04:56:53.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985765+0000", 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000337812 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:172433 10.a e316/293 10:5ddcd198:::benchmark_data_smithi082_223129_object66984:head v 316'10910, mlcod=316'10908)", 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.980603+0000", 2023-10-09T04:56:53.026 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.017674639999999998, 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.018066885000000001, 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980603+0000", 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.027 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980603+0000", 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980602+0000", 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.028 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980612+0000", 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0705e-05 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980625+0000", 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2500000000000001e-05 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.980646+0000", 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.0636999999999999e-05 2023-10-09T04:56:53.029 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.984507+0000", 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.003861284 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.984525+0000", 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.8614000000000002e-05 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.030 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:172429 10.6 e316/313 10:67b580b5:::benchmark_data_smithi082_223129_object66980:head v 316'10817, mlcod=316'10814)", 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.981412+0000", 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.016865092000000002, 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017383027999999998, 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.031 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981412+0000", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981412+0000", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981411+0000", 2023-10-09T04:56:53.032 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981422+0000", 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0101e-05 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981433+0000", 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1858000000000001e-05 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.033 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.981441+0000", 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.6490000000000008e-06 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.983939+0000", 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0024975050000000001 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.034 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.984021+0000", 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.1988000000000002e-05 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:172438 10.1 10:82681420:::benchmark_data_smithi082_223129_object66989:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.985302+0000", 2023-10-09T04:56:53.035 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.012975462, 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013557496, 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 172438 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.036 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985302+0000", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985302+0000", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985300+0000", 2023-10-09T04:56:53.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985306+0000", 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.8599999999999998e-06 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985316+0000", 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0513000000000001e-05 2023-10-09T04:56:53.038 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985324+0000", 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9240000000000007e-06 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985408+0000", 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3693e-05 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.039 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985563+0000", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015502199999999999 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.985701+0000", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013831 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.995731+0000", 2023-10-09T04:56:53.040 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.01002919 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:172432 10.2 e316/313)", 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.990267+0000", 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0080101510000000001, 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0086669539999999993, 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.041 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990267+0000", 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990267+0000", 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.042 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990266+0000", 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990273+0000", 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6229999999999999e-06 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.043 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990290+0000", 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7008999999999999e-05 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990299+0000", 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1649999999999995e-06 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.044 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:172432 10.2 e316/313)", 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.990377+0000", 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0079005240000000008, 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0086025530000000006, 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990377+0000", 2023-10-09T04:56:53.045 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990377+0000", 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990377+0000", 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:53.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990379+0000", 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6740000000000001e-06 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990386+0000", 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2859999999999996e-06 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990391+0000", 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:172439 10.4 10.a96a0474 (undecoded) ondisk+write+known_if_redirected e316)", 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.990411+0000", 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0078658579999999999, 2023-10-09T04:56:53.048 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0086149350000000006, 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "queued for pg", 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 172439 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.049 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990411+0000", 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990411+0000", 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990410+0000", 2023-10-09T04:56:53.050 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990414+0000", 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7489999999999998e-06 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990423+0000", 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.6149999999999997e-06 2023-10-09T04:56:53.051 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990429+0000", 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.852e-06 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:172441 10.1 10:86afe5ad:::benchmark_data_smithi082_223129_object66992:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.990442+0000", 2023-10-09T04:56:53.052 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0078355419999999992, 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0086413470000000006, 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 172441 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990442+0000", 2023-10-09T04:56:53.053 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990442+0000", 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990441+0000", 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.054 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990444+0000", 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9040000000000001e-06 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990450+0000", 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9429999999999997e-06 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990455+0000", 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.1760000000000001e-06 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.990889+0000", 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00043385100000000002 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.991026+0000", 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000136921 2023-10-09T04:56:53.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.991154+0000", 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012782700000000001 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996160+0000", 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0050061760000000002 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.057 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:172442 10.c e316/313 10:3a17a2b6:::benchmark_data_smithi082_223129_object66993:head v 316'10734, mlcod=316'10732)", 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:52.994837+0000", 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 0.0034399600000000002, 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0043235440000000003, 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:56:53.058 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994837+0000", 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994837+0000", 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994836+0000", 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:53.059 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994846+0000", 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4800000000000007e-06 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994860+0000", 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.45e-05 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.060 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.994870+0000", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8959999999999998e-06 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996200+0000", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0013297579999999999 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:52.996224+0000", 2023-10-09T04:56:53.061 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4535000000000001e-05 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: ], 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout: "num_ops": 14 2023-10-09T04:56:53.062 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:56:53.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:53.083 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:53.094 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:56:53.125 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:53.145 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:56:53.181 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:56:53.186 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:53.211 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:53.221 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:56:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:53 smithi082 bash[77063]: cluster 2023-10-09T04:56:52.953164+0000 mon.a (mon.0) 1855 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/124182 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:56:53.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:53 smithi082 bash[77063]: cluster 2023-10-09T04:56:52.953211+0000 mon.a (mon.0) 1856 : cluster 1 Cluster is now healthy 2023-10-09T04:56:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:53 smithi082 bash[79733]: cluster 2023-10-09T04:56:52.953164+0000 mon.a (mon.0) 1855 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/124182 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:56:53.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:53 smithi082 bash[79733]: cluster 2023-10-09T04:56:52.953211+0000 mon.a (mon.0) 1856 : cluster 1 Cluster is now healthy 2023-10-09T04:56:53.256 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:56:53.298 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:56:53.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:53.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:53 smithi153 bash[59565]: cluster 2023-10-09T04:56:52.953164+0000 mon.a (mon.0) 1855 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/124182 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:56:53.322 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:53 smithi153 bash[59565]: cluster 2023-10-09T04:56:52.953211+0000 mon.a (mon.0) 1856 : cluster 1 Cluster is now healthy 2023-10-09T04:56:53.339 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:56:53.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:53.372 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.387 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:56:53.451 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:53.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:53.582 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:53.613 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:53.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:53.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:53.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:53.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:53.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:53.999 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:54.132 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:54.141 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:54 smithi153 bash[59565]: cluster 2023-10-09T04:56:52.901877+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v1417: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 19685/119673 objects misplaced (16.449%); 160 KiB/s, 6 objects/s recovering 2023-10-09T04:56:54.142 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:54 smithi153 bash[59565]: audit 2023-10-09T04:56:53.021555+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.82:0/2393003089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:54.169 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:54.274 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:54.289 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:54 smithi082 bash[77063]: cluster 2023-10-09T04:56:52.901877+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v1417: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 19685/119673 objects misplaced (16.449%); 160 KiB/s, 6 objects/s recovering 2023-10-09T04:56:54.289 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:54 smithi082 bash[77063]: audit 2023-10-09T04:56:53.021555+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.82:0/2393003089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:54.289 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:54 smithi082 bash[79733]: cluster 2023-10-09T04:56:52.901877+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v1417: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 19685/119673 objects misplaced (16.449%); 160 KiB/s, 6 objects/s recovering 2023-10-09T04:56:54.289 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:54 smithi082 bash[79733]: audit 2023-10-09T04:56:53.021555+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.82:0/2393003089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:54.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:54.400 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.423 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:54.526 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:54.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:54.688 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:54.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:54.818 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:54.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.958 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:54.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:54.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:55.066 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:55.102 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:55.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:55.260 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:55.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:55.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:55.406 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:55.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:55.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:55.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:55.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:55.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:55.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:55.938 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:55.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:55.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:56.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:56.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:56.206 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:56.214 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:56 smithi153 bash[59565]: cluster 2023-10-09T04:56:54.902397+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 19177/116106 objects misplaced (16.517%); 1.1 MiB/s, 20 objects/s recovering 2023-10-09T04:56:56.239 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:56.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:56.354 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:56 smithi082 bash[77063]: cluster 2023-10-09T04:56:54.902397+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 19177/116106 objects misplaced (16.517%); 1.1 MiB/s, 20 objects/s recovering 2023-10-09T04:56:56.355 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:56 smithi082 bash[79733]: cluster 2023-10-09T04:56:54.902397+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 19177/116106 objects misplaced (16.517%); 1.1 MiB/s, 20 objects/s recovering 2023-10-09T04:56:56.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:56.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:56.491 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:56.525 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:56.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:56:56.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:56.659 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:56.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:56.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:56.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:56.953 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:56.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:57.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:57.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:57.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:57.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:57.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:57.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:57.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:57.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:57.596 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:57.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:57.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:57.772 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:57.877 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:57.925 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:57.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:58.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:58.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:58.167 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.208 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:58.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.313 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:56:58 smithi082 bash[77063]: cluster 2023-10-09T04:56:56.902891+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 18767/114303 objects misplaced (16.419%); 987 KiB/s, 18 objects/s recovering 2023-10-09T04:56:58.313 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:56:58 smithi082 bash[79733]: cluster 2023-10-09T04:56:56.902891+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 18767/114303 objects misplaced (16.419%); 987 KiB/s, 18 objects/s recovering 2023-10-09T04:56:58.351 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:58.456 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:58.466 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:56:58 smithi153 bash[59565]: cluster 2023-10-09T04:56:56.902891+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 18767/114303 objects misplaced (16.419%); 987 KiB/s, 18 objects/s recovering 2023-10-09T04:56:58.483 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:58.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:58.586 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:58.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:58.728 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:56:58.766 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:58.871 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:58.896 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:58.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:59.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:59.146 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:59.181 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:59.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.323 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:59.431 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:59.467 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:59.571 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:59.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:56:59.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:56:59.699 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:56:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:56:59] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-10-09T04:56:59.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.758 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:59.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:59.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:59.893 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:56:59.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:59.999 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:00.025 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:00.131 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:00.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:00.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[77063]: cluster 2023-10-09T04:56:58.903500+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 17443/106383 objects misplaced (16.396%); 1.5 MiB/s, 25 objects/s recovering 2023-10-09T04:57:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[77063]: audit 2023-10-09T04:56:59.301825+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:00.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[77063]: audit 2023-10-09T04:57:00.060499+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[79733]: cluster 2023-10-09T04:56:58.903500+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 17443/106383 objects misplaced (16.396%); 1.5 MiB/s, 25 objects/s recovering 2023-10-09T04:57:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[79733]: audit 2023-10-09T04:56:59.301825+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:00.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:00 smithi082 bash[79733]: audit 2023-10-09T04:57:00.060499+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:00.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:00.281 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:00 smithi153 bash[59565]: cluster 2023-10-09T04:56:58.903500+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 17443/106383 objects misplaced (16.396%); 1.5 MiB/s, 25 objects/s recovering 2023-10-09T04:57:00.282 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:00 smithi153 bash[59565]: audit 2023-10-09T04:56:59.301825+0000 mon.a (mon.0) 1857 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:00.282 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:00 smithi153 bash[59565]: audit 2023-10-09T04:57:00.060499+0000 mon.a (mon.0) 1858 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:00.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:00.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:00.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:00.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:00.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:00.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:00.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:00.671 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:00.695 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:00.801 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:00.835 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:00.941 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:00.968 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:00.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:01.077 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:01.116 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:01.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:01.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:01 smithi153 bash[59565]: audit 2023-10-09T04:57:00.595679+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.82:0/947655692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:01 smithi153 bash[59565]: audit 2023-10-09T04:57:00.596265+0000 mon.a (mon.0) 1859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.209 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:01 smithi153 bash[59565]: audit 2023-10-09T04:57:00.854544+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:01.219 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:57:01.220 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:01.366 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:01.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[79733]: audit 2023-10-09T04:57:00.595679+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.82:0/947655692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[79733]: audit 2023-10-09T04:57:00.596265+0000 mon.a (mon.0) 1859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[79733]: audit 2023-10-09T04:57:00.854544+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[77063]: audit 2023-10-09T04:57:00.595679+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.82:0/947655692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[77063]: audit 2023-10-09T04:57:00.596265+0000 mon.a (mon.0) 1859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:01.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:01 smithi082 bash[77063]: audit 2023-10-09T04:57:00.854544+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:57:01.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.521 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:01.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:01.656 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:01.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.793 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:01.895 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:01.935 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:01.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:02.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:02.062 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:02.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:02.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:02.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:02.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:02.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:02.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:00.904067+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17249/105198 objects misplaced (16.397%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:57:02.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:01.186521+0000 mon.a (mon.0) 1861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:02.453 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:01.205535+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: audit 2023-10-09T04:57:01.209146+0000 mon.a (mon.0) 1862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:01.209328+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:01.210493+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[77063]: cluster 2023-10-09T04:57:01.220851+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:00.904067+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17249/105198 objects misplaced (16.397%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:01.186521+0000 mon.a (mon.0) 1861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:01.205535+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.454 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: audit 2023-10-09T04:57:01.209146+0000 mon.a (mon.0) 1862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:02.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:01.209328+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T04:57:02.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:01.210493+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.455 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:02 smithi082 bash[79733]: cluster 2023-10-09T04:57:01.220851+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.482 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:02.588 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:02.627 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:00.904067+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17249/105198 objects misplaced (16.397%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:57:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:01.186521+0000 mon.a (mon.0) 1861 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:01.205535+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: audit 2023-10-09T04:57:01.209146+0000 mon.a (mon.0) 1862 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:01.209328+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-10-09T04:57:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:01.210493+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:02 smithi153 bash[59565]: cluster 2023-10-09T04:57:01.220851+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T04:57:02.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:02.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:02.894 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:02.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:02.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:03.031 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:03.138 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:03.156 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:03.156 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:03.167 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:03.271 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:03.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:03.407 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:03.439 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:03.542 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:03.582 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:03.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:03.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:03.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:03.854 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:03.960 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:03.989 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:03.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:04.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:04.117 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:04.217 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:04 smithi082 bash[77063]: cluster 2023-10-09T04:57:02.904832+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 16296/100416 objects misplaced (16.228%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:04.220 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:04 smithi082 bash[79733]: cluster 2023-10-09T04:57:02.904832+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 16296/100416 objects misplaced (16.228%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:04.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:04.362 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.371 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:04 smithi153 bash[59565]: cluster 2023-10-09T04:57:02.904832+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 16296/100416 objects misplaced (16.228%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:04.387 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:04.491 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:04.514 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:04.617 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.649 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:04.735 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:57:04.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:04.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:04.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:04.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:04.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:04.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:05.092 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.118 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:05.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:05.219 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.255 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:05.263 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:05.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:05.412 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:05.519 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.559 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:05.664 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.698 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:05.806 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:05.834 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:05.937 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:05.967 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:05.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:06.071 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:06.102 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:06.210 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:06.221 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:06 smithi082 bash[77063]: cluster 2023-10-09T04:57:04.905439+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15792/96843 objects misplaced (16.307%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:06.221 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:06 smithi082 bash[79733]: cluster 2023-10-09T04:57:04.905439+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15792/96843 objects misplaced (16.307%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:06.248 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:06.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:06.397 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:06 smithi153 bash[59565]: cluster 2023-10-09T04:57:04.905439+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15792/96843 objects misplaced (16.307%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:06.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:06.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:06.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:06.687 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:06.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:06.825 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:06.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:06.932 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:06.972 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:06.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:06.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:07.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:07.111 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:07.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:07.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:07.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:07.391 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:07.495 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:07.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:07.625 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:07.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:07.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:07.790 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:07.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:07.899 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:07.945 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:07.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:08.048 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:08.078 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:08.186 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[77063]: cluster 2023-10-09T04:57:06.906059+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 15428/95220 objects misplaced (16.202%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[77063]: audit 2023-10-09T04:57:07.527911+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[77063]: audit 2023-10-09T04:57:07.539049+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[77063]: audit 2023-10-09T04:57:07.581892+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[77063]: audit 2023-10-09T04:57:07.594784+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[79733]: cluster 2023-10-09T04:57:06.906059+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 15428/95220 objects misplaced (16.202%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[79733]: audit 2023-10-09T04:57:07.527911+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.229 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[79733]: audit 2023-10-09T04:57:07.539049+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[79733]: audit 2023-10-09T04:57:07.581892+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:08 smithi082 bash[79733]: audit 2023-10-09T04:57:07.594784+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:08.331 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:08.458 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:08.483 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:08.588 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:08 smithi153 bash[59565]: cluster 2023-10-09T04:57:06.906059+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 15428/95220 objects misplaced (16.202%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:08.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:08 smithi153 bash[59565]: audit 2023-10-09T04:57:07.527911+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:08 smithi153 bash[59565]: audit 2023-10-09T04:57:07.539049+0000 mon.a (mon.0) 1865 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:08 smithi153 bash[59565]: audit 2023-10-09T04:57:07.581892+0000 mon.a (mon.0) 1866 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.598 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:08 smithi153 bash[59565]: audit 2023-10-09T04:57:07.594784+0000 mon.a (mon.0) 1867 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:08.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:08.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:08.761 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:08.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:08.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:09.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:09.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:09.147 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:09.179 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:09.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:09.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:09.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:09.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[77063]: audit 2023-10-09T04:57:08.248249+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:09.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[77063]: audit 2023-10-09T04:57:08.253141+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:09.424 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[77063]: audit 2023-10-09T04:57:08.264442+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[79733]: audit 2023-10-09T04:57:08.248249+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[79733]: audit 2023-10-09T04:57:08.253141+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[79733]: audit 2023-10-09T04:57:08.264442+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:09 smithi153 bash[59565]: audit 2023-10-09T04:57:08.248249+0000 mon.a (mon.0) 1868 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:09 smithi153 bash[59565]: audit 2023-10-09T04:57:08.253141+0000 mon.a (mon.0) 1869 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:09 smithi153 bash[59565]: audit 2023-10-09T04:57:08.264442+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:09.439 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:09.542 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:09.572 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:09.676 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:09.700 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:09.802 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:09.832 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:09.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:09.969 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:09.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:09] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-10-09T04:57:10.072 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:10.106 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:10.212 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:10.244 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:10.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:10.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:10.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:10 smithi153 bash[59565]: cluster 2023-10-09T04:57:08.908013+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14342/88545 objects misplaced (16.197%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:10.471 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:10.500 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:10.604 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:10.625 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:10.669 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:10 smithi082 bash[77063]: cluster 2023-10-09T04:57:08.908013+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14342/88545 objects misplaced (16.197%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:10.728 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:10.759 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:10.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:10.893 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:10.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:10.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:10.972 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:10 smithi082 bash[79733]: cluster 2023-10-09T04:57:08.908013+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 14342/88545 objects misplaced (16.197%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:10.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:11.029 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:11.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:11.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:11.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:11.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:11.269 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:11.286 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:11 smithi153 bash[59565]: cluster 2023-10-09T04:57:10.908738+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 14176/87474 objects misplaced (16.206%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:11.308 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:11.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:11.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:11.443 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:11.548 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:11.557 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:11 smithi082 bash[77063]: cluster 2023-10-09T04:57:10.908738+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 14176/87474 objects misplaced (16.206%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:11.557 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:11 smithi082 bash[79733]: cluster 2023-10-09T04:57:10.908738+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 14176/87474 objects misplaced (16.206%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:11.582 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:11.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:11.689 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:11.723 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:11.831 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:11.861 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:11.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:11.968 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:12.001 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:12.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:12.131 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:12.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:12.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:12.394 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:12.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:12.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:12.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:12.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:12.690 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:12.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:12.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:12.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:12.936 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:12.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:12.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:13.070 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.098 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:13.314 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:13.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:13.447 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.474 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:13.576 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.606 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:13.712 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.744 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:13.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:13.874 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:13.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:13.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:14.007 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:14.112 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.121 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:13 smithi082 bash[77063]: cluster 2023-10-09T04:57:12.909369+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 13252/83022 objects misplaced (15.962%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:14.121 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:13 smithi082 bash[79733]: cluster 2023-10-09T04:57:12.909369+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 13252/83022 objects misplaced (15.962%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:14.140 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:14.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:13 smithi153 bash[59565]: cluster 2023-10-09T04:57:12.909369+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 13252/83022 objects misplaced (15.962%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:14.243 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.268 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:14.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:14.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:14.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:14.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:14.644 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:14.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:14.783 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.812 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:14.916 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:14.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:14.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:15.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:15.080 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:15.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:15.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:15.315 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:15.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:15.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[77063]: audit 2023-10-09T04:57:14.307796+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:15.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[77063]: audit 2023-10-09T04:57:14.318706+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:15.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[77063]: cluster 2023-10-09T04:57:14.909907+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 12812/79593 objects misplaced (16.097%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:15.365 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[79733]: audit 2023-10-09T04:57:14.307796+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:15.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[79733]: audit 2023-10-09T04:57:14.318706+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:15.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:15 smithi082 bash[79733]: cluster 2023-10-09T04:57:14.909907+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 12812/79593 objects misplaced (16.097%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:15.448 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:15.470 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:15.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:15.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:15 smithi153 bash[59565]: audit 2023-10-09T04:57:14.307796+0000 mon.a (mon.0) 1871 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:15.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:15 smithi153 bash[59565]: audit 2023-10-09T04:57:14.318706+0000 mon.a (mon.0) 1872 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:15.582 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:15 smithi153 bash[59565]: cluster 2023-10-09T04:57:14.909907+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v1429: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 12812/79593 objects misplaced (16.097%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:15.603 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:15.706 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:15.746 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:15.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:15.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:15.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:15.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.028 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:16.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:16.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:16.301 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:16.351 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.2 primary-affinity to 0.0755039 (849482) 2023-10-09T04:57:16.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:16.428 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:16.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.543 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:16 smithi082 bash[77063]: audit 2023-10-09T04:57:16.007561+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.82:0/897919127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.543 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:16 smithi082 bash[77063]: audit 2023-10-09T04:57:16.008327+0000 mon.a (mon.0) 1873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.543 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:16 smithi082 bash[79733]: audit 2023-10-09T04:57:16.007561+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.82:0/897919127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.543 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:16 smithi082 bash[79733]: audit 2023-10-09T04:57:16.008327+0000 mon.a (mon.0) 1873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.556 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:16.659 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:16 smithi153 bash[59565]: audit 2023-10-09T04:57:16.007561+0000 mon.c (mon.2) 140 : audit 1 from='client.? 172.21.15.82:0/897919127' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:16 smithi153 bash[59565]: audit 2023-10-09T04:57:16.008327+0000 mon.a (mon.0) 1873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]: dispatch 2023-10-09T04:57:16.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:16.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:16.815 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:16.918 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:16.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:16.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:17.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:17.075 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:17.178 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:17.205 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:17.308 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:17.332 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:17.434 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:17.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:17.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:17.520 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[77063]: cluster 2023-10-09T04:57:16.329690+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T04:57:17.520 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[77063]: audit 2023-10-09T04:57:16.332723+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]': finished 2023-10-09T04:57:17.520 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[77063]: cluster 2023-10-09T04:57:16.332805+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T04:57:17.520 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[77063]: cluster 2023-10-09T04:57:16.910612+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 12455/77949 objects misplaced (15.978%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:17.521 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[79733]: cluster 2023-10-09T04:57:16.329690+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T04:57:17.521 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[79733]: audit 2023-10-09T04:57:16.332723+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]': finished 2023-10-09T04:57:17.521 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[79733]: cluster 2023-10-09T04:57:16.332805+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T04:57:17.521 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:17 smithi082 bash[79733]: cluster 2023-10-09T04:57:16.910612+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 12455/77949 objects misplaced (15.978%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:17.544 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:57:17.544 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:57:17.565 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:17.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:17 smithi153 bash[59565]: cluster 2023-10-09T04:57:16.329690+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T04:57:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:17 smithi153 bash[59565]: audit 2023-10-09T04:57:16.332723+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.07550390487827374}]': finished 2023-10-09T04:57:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:17 smithi153 bash[59565]: cluster 2023-10-09T04:57:16.332805+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-10-09T04:57:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:17 smithi153 bash[59565]: cluster 2023-10-09T04:57:16.910612+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 12455/77949 objects misplaced (15.978%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:17.711 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:17.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:17.842 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:17.869 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:17.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:17.972 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:18.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:18.111 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:18.160 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:18.265 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:18.290 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:18.397 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.342260+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.346383+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.349173+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.349270+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.349575+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.350027+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.352353+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.359536+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:18 smithi153 bash[59565]: cluster 2023-10-09T04:57:17.359540+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.429 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:18.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:18.535 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:18.562 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:18.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.342260+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.346383+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.349173+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.349270+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.349575+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.350027+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.677 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.352353+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.359536+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[77063]: cluster 2023-10-09T04:57:17.359540+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.342260+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.346383+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.349173+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.349270+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.349575+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.350027+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.352353+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.678 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.359536+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.679 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:18 smithi082 bash[79733]: cluster 2023-10-09T04:57:17.359540+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T04:57:18.697 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:18.801 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:18.826 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:18.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:18.981 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:18.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:19.087 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:19.122 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:19.228 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:19.262 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:19.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:19.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:19.378 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:19 smithi153 bash[59565]: cluster 2023-10-09T04:57:18.911166+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11389/72330 objects misplaced (15.746%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:19.404 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:19.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:19.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:19.550 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:19.657 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:19.684 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:19.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:19 smithi082 bash[77063]: cluster 2023-10-09T04:57:18.911166+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11389/72330 objects misplaced (15.746%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:19.700 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:19] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.33.4" 2023-10-09T04:57:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:19 smithi082 bash[79733]: cluster 2023-10-09T04:57:18.911166+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11389/72330 objects misplaced (15.746%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:19.787 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:19.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:19.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:19.959 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:19.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:20.063 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:20.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:20.199 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:20.231 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:20.342 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:20.367 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:20.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:20.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:20.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:20.607 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:20.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:20.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:20.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:20.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:20.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:57:20.874 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:20.903 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:20.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:21.008 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:21.036 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:21.105 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:57:21.141 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:21.146 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:57:21.176 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:21.198 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:57:21.244 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:57:21.277 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:57:21.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:21.287 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:21 smithi153 bash[59565]: audit 2023-10-09T04:57:21.043067+0000 mon.a (mon.0) 1877 : audit 0 from='client.? 172.21.15.82:0/3056106576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:21.291 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:21.304 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:21.311 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:57:21.351 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:57:21.394 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7: {} 2023-10-09T04:57:21.405 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.405 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.405 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.405 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.405 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.406 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.406 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.406 INFO:teuthology.orchestra.run.smithi082.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:57:21.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:21.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:21.450 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:21 smithi082 bash[77063]: audit 2023-10-09T04:57:21.043067+0000 mon.a (mon.0) 1877 : audit 0 from='client.? 172.21.15.82:0/3056106576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:21.451 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:21 smithi082 bash[79733]: audit 2023-10-09T04:57:21.043067+0000 mon.a (mon.0) 1877 : audit 0 from='client.? 172.21.15.82:0/3056106576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:21.545 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:21.574 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:21.680 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:21.729 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:21.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:21.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:21.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:21.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:22.001 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:22.109 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:22.165 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:22.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:22.195 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:22 smithi082 bash[77063]: cluster 2023-10-09T04:57:20.912251+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 11237/70611 objects misplaced (15.914%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:22.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:22 smithi082 bash[79733]: cluster 2023-10-09T04:57:20.912251+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 11237/70611 objects misplaced (15.914%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:22.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.384 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:22.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:22 smithi153 bash[59565]: cluster 2023-10-09T04:57:20.912251+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 11237/70611 objects misplaced (15.914%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:57:22.488 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.534 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:22.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:22.671 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:22.780 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.815 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:22.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:22.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:22.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:23.100 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:23.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:23.237 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:23.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:23.343 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:23.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:23.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.496 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.602 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:23.629 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:23.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:23.732 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:23.753 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:23.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.885 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:23.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:24.015 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:24.118 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:24.168 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:24.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:24.271 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:24.280 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:24 smithi082 bash[77063]: cluster 2023-10-09T04:57:22.912829+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10390/66984 objects misplaced (15.511%); 936 KiB/s, 14 objects/s recovering 2023-10-09T04:57:24.280 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:24 smithi082 bash[79733]: cluster 2023-10-09T04:57:22.912829+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10390/66984 objects misplaced (15.511%); 936 KiB/s, 14 objects/s recovering 2023-10-09T04:57:24.299 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:24.404 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:24 smithi153 bash[59565]: cluster 2023-10-09T04:57:22.912829+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 10390/66984 objects misplaced (15.511%); 936 KiB/s, 14 objects/s recovering 2023-10-09T04:57:24.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:24.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:24.536 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:24.564 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:24.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:24.668 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:24.697 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:24.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:24.774 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:57:24.774 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:24.774 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:57:24.774 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:57:24.775 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:57:24.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:57:24.803 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:24.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:24.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:24.963 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:24.997 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:25.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:25.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:25.146 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:25.251 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:25.277 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:25.383 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:25.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:25.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.552 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:25.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:25.653 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:25.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:57:25.676 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:25.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:25.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:25.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:25.948 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:25.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:26.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:26.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:26.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:26.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:26.230 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:26.336 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:26.360 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:26.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:26 smithi082 bash[77063]: cluster 2023-10-09T04:57:24.913316+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v1436: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9963/63540 objects misplaced (15.680%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:26.468 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:26 smithi082 bash[79733]: cluster 2023-10-09T04:57:24.913316+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v1436: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9963/63540 objects misplaced (15.680%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:26.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:26.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:26.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:26.625 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:26 smithi153 bash[59565]: cluster 2023-10-09T04:57:24.913316+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v1436: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9963/63540 objects misplaced (15.680%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:57:26.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:26.762 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:26.788 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:26.892 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:26.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:26.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:27.035 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:27.078 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:27.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:27.184 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.217 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:27.234 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:57:27.320 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.355 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:27.458 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:27.467 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:27 smithi153 bash[59565]: audit 2023-10-09T04:57:27.065514+0000 mon.a (mon.0) 1878 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:27.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:27 smithi082 bash[77063]: audit 2023-10-09T04:57:27.065514+0000 mon.a (mon.0) 1878 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:27.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:27 smithi082 bash[79733]: audit 2023-10-09T04:57:27.065514+0000 mon.a (mon.0) 1878 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:27.494 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:27.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:27.598 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:27.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:27.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:27.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:27.869 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:27.914 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:27.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:28.015 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:28.156 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:28.295 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:28.331 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:28.438 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: cluster 2023-10-09T04:57:26.913968+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 9584/61215 objects misplaced (15.656%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: cluster 2023-10-09T04:57:27.206831+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: cluster 2023-10-09T04:57:27.219597+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: audit 2023-10-09T04:57:27.224276+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: cluster 2023-10-09T04:57:27.224338+0000 mon.a (mon.0) 1881 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T04:57:28.447 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[77063]: cluster 2023-10-09T04:57:27.225048+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: cluster 2023-10-09T04:57:26.913968+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 9584/61215 objects misplaced (15.656%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: cluster 2023-10-09T04:57:27.206831+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: cluster 2023-10-09T04:57:27.219597+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: audit 2023-10-09T04:57:27.224276+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: cluster 2023-10-09T04:57:27.224338+0000 mon.a (mon.0) 1881 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T04:57:28.448 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:28 smithi082 bash[79733]: cluster 2023-10-09T04:57:27.225048+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.473 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:28.578 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.607 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: cluster 2023-10-09T04:57:26.913968+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 9584/61215 objects misplaced (15.656%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: cluster 2023-10-09T04:57:27.206831+0000 mon.a (mon.0) 1879 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: cluster 2023-10-09T04:57:27.219597+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: audit 2023-10-09T04:57:27.224276+0000 mon.a (mon.0) 1880 : audit 1 from='client.? 172.21.15.82:0/1118424191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: cluster 2023-10-09T04:57:27.224338+0000 mon.a (mon.0) 1881 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-10-09T04:57:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:28 smithi153 bash[59565]: cluster 2023-10-09T04:57:27.225048+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T04:57:28.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:28.868 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:28.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:29.000 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.033 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:29.102 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:29.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:29.173 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:29.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:29.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.315 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:29.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:29.557 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:29.582 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:29.623 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:57:29.691 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:29.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:29.837 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:29.884 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:29.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:29] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-09T04:57:29.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:30.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:30.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:30.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:30.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:30.425 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:30 smithi082 bash[77063]: cluster 2023-10-09T04:57:28.914590+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8408/54246 objects misplaced (15.500%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:30.426 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:30 smithi082 bash[77063]: audit 2023-10-09T04:57:29.306982+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:30.426 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:30 smithi082 bash[79733]: cluster 2023-10-09T04:57:28.914590+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8408/54246 objects misplaced (15.500%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:30.426 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:30 smithi082 bash[79733]: audit 2023-10-09T04:57:29.306982+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:30.443 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:30.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:30.549 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:30.558 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:30 smithi153 bash[59565]: cluster 2023-10-09T04:57:28.914590+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8408/54246 objects misplaced (15.500%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:30.559 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:30 smithi153 bash[59565]: audit 2023-10-09T04:57:29.306982+0000 mon.a (mon.0) 1882 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:30.572 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:30.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:30.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:57:30.701 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:30.804 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.826 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:30.930 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.968 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:30.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:31.073 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:31.215 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.223 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:31.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:31.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:31.498 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.519 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:31.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:31.758 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.786 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:31.890 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:31.930 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:31.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:32.035 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:32.060 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:32.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:32.164 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:32.197 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:32.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:32.316 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:32 smithi082 bash[77063]: cluster 2023-10-09T04:57:30.915247+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 8223/52629 objects misplaced (15.624%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:32.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:32 smithi082 bash[79733]: cluster 2023-10-09T04:57:30.915247+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 8223/52629 objects misplaced (15.624%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:32 smithi153 bash[59565]: cluster 2023-10-09T04:57:30.915247+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 8223/52629 objects misplaced (15.624%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:32.439 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:32.477 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:32.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:32.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:32.632 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:32.738 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:32.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:32.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:32.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:32.900 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:32.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:33.004 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:33.045 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:33.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.189 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:33.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.325 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:33.428 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:33.464 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:33.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:33.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:33.708 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.755 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:33.862 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:33.886 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:33.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:33.994 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:34.019 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:34.131 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:34.164 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:34.270 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:34.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:34.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:34.406 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:34 smithi153 bash[59565]: cluster 2023-10-09T04:57:32.916101+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v1441: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1016 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7257/48615 objects misplaced (14.927%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:34.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:34.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:34 smithi082 bash[77063]: cluster 2023-10-09T04:57:32.916101+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v1441: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1016 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7257/48615 objects misplaced (14.927%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:34 smithi082 bash[79733]: cluster 2023-10-09T04:57:32.916101+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v1441: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 1016 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7257/48615 objects misplaced (14.927%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:34.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:34.568 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:34.674 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:34.708 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:57:34.708 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "" 2023-10-09T04:57:34.708 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:57:34.712 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:34.816 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:34.845 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:34.951 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:34.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:34.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:35.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:35.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:35.233 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:35.264 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:35.367 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:35.398 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:35.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:35.526 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:35.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:35.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.801 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:35.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.932 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:35.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout: "size": 20, 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 600, 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [ 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.993 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162034 9.4 9.c3d51c24 (undecoded) ondisk+write+known_if_redirected e279)", 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:13.134404+0000", 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 382.85529087200001, 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96180686800000004, 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162034 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.994 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134404+0000", 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134404+0000", 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:35.995 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134402+0000", 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134408+0000", 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3720000000000004e-06 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.996 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134417+0000", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.4720000000000001e-06 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134424+0000", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134467+0000", 2023-10-09T04:57:35.997 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3677000000000002e-05 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.134479+0000", 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1919999999999999e-05 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096199+0000", 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.96171937799999996 2023-10-09T04:57:35.998 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096210+0000", 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.1831000000000001e-05 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162039 9.4 9.6d7d24a4 (undecoded) ondisk+write+known_if_redirected e279)", 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:13.141343+0000", 2023-10-09T04:57:35.999 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 382.84835185899999, 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.95495172699999997, 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162039 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.000 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141343+0000", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141343+0000", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141341+0000", 2023-10-09T04:57:36.001 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141350+0000", 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.0820000000000005e-06 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141367+0000", 2023-10-09T04:57:36.002 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7116999999999999e-05 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141376+0000", 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.2909999999999996e-06 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141428+0000", 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.2141000000000001e-05 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.003 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.141442+0000", 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3336000000000001e-05 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096284+0000", 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.95484221800000002 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.004 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096294+0000", 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0232e-05 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162045 9.4 9.12f86244 (undecoded) ondisk+write+known_if_redirected e279)", 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:13.146480+0000", 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 382.843214381, 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94984457300000003, 2023-10-09T04:57:36.005 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162045 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146480+0000", 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.006 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146480+0000", 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146479+0000", 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.007 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146485+0000", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146497+0000", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2537e-05 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146505+0000", 2023-10-09T04:57:36.008 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.2130000000000002e-06 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146630+0000", 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000125246 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.146641+0000", 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.098e-05 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.009 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096316+0000", 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.94967548199999996 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096325+0000", 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2120000000000003e-06 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.010 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.35685.0:162072 9.4 9.6eda0624 (undecoded) ondisk+write+known_if_redirected e279)", 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:51:13.209586+0000", 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 382.78010799600003, 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.88676779999999999, 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.35685", 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/3534347120", 2023-10-09T04:57:36.011 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 162072 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209586+0000", 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209586+0000", 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.012 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209585+0000", 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209592+0000", 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0040000000000002e-06 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.013 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209607+0000", 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4881e-05 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209619+0000", 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2228999999999999e-05 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209668+0000", 2023-10-09T04:57:36.014 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8603999999999997e-05 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:13.209677+0000", 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1439999999999992e-06 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096347+0000", 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.88667049399999998 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.015 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:51:14.096354+0000", 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.9149999999999999e-06 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:21627 10.8 10:13330b69:::benchmark_data_smithi082_223129_object21626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:53:56.405506+0000", 2023-10-09T04:57:36.016 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 219.58418884899999, 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.427577506, 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 21627 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.017 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405506+0000", 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405506+0000", 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405504+0000", 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:36.018 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405530+0000", 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5778999999999998e-05 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405543+0000", 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3191e-05 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.019 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405552+0000", 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.9449999999999999e-06 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405600+0000", 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.8140999999999999e-05 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.020 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405739+0000", 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013933399999999999 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.405882+0000", 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014214300000000001 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.406971+0000", 2023-10-09T04:57:36.021 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00108895 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832857+0000", 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42588635800000002 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832941+0000", 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3702000000000006e-05 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.022 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.833012+0000", 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1341999999999996e-05 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.833083+0000", 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1120999999999999e-05 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.023 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:21633 10.d e302/293 10:baff69f7:::benchmark_data_smithi082_223129_object21632:head v 302'1300, mlcod=302'1296)", 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:53:56.407898+0000", 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 219.581796771, 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42485086799999999, 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.407898+0000", 2023-10-09T04:57:36.024 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.407898+0000", 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.407897+0000", 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.025 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.407973+0000", 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5771000000000003e-05 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.407994+0000", 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.1274999999999999e-05 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.026 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408001+0000", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.6760000000000001e-06 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408045+0000", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4409000000000001e-05 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408067+0000", 2023-10-09T04:57:36.027 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.2041000000000001e-05 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.409676+0000", 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001608492 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832748+0000", 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42307288300000001 2023-10-09T04:57:36.028 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop(client.45806.0:21630 10.a e302/293 10:524a61f3:::benchmark_data_smithi082_223129_object21629:head v 302'1409, mlcod=302'1407)", 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:53:56.408013+0000", 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 219.58168103, 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42476334500000001, 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.029 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408013+0000", 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408013+0000", 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.030 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408013+0000", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408058+0000", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4870000000000002e-05 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408065+0000", 2023-10-09T04:57:36.031 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.4200000000000001e-06 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408070+0000", 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.0660000000000003e-06 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408107+0000", 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6748999999999997e-05 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.032 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.408126+0000", 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.895e-05 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832729+0000", 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42460294900000001 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.033 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832777+0000", 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.7707000000000002e-05 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:21627 10.8 e302/293 ondisk, result = 0)", 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:53:56.409717+0000", 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 219.57997721300001, 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.423147404, 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.034 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.409717+0000", 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.409717+0000", 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.035 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.409716+0000", 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.409729+0000", 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.255e-05 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.036 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832731+0000", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42300223999999997 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832745+0000", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.412e-05 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832833+0000", 2023-10-09T04:57:36.037 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.8091999999999996e-05 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832848+0000", 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4365999999999999e-05 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832865+0000", 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6798000000000001e-05 2023-10-09T04:57:36.038 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_repop_reply(client.45806.0:21627 10.8 e302/293 ondisk, result = 0)", 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:53:56.410379+0000", 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 219.57931536199999, 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42277487000000002, 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "started", 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.039 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.410379+0000", 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.410379+0000", 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.040 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.410378+0000", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.410384+0000", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.519e-06 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832771+0000", 2023-10-09T04:57:36.041 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.42238692999999999 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832780+0000", 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.85e-06 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832924+0000", 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014476699999999999 2023-10-09T04:57:36.042 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.832934+0000", 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.7529999999999996e-06 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:53:56.833154+0000", 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00021984700000000001 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.043 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:59978 10.8 10:1809c65c:::benchmark_data_smithi082_223129_object59977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:54:25.303052+0000", 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 190.68664226199999, 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28537944700000001, 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.044 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 59978 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303052+0000", 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303052+0000", 2023-10-09T04:57:36.045 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303051+0000", 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303082+0000", 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0590999999999999e-05 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.046 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303090+0000", 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.8939999999999993e-06 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303094+0000", 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3560000000000003e-06 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.047 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303284+0000", 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018965599999999999 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303428+0000", 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000143869 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303615+0000", 2023-10-09T04:57:36.048 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018727699999999999 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.308393+0000", 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0047781459999999996 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.308923+0000", 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00052951799999999998 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.049 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588270+0000", 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.279347287 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588385+0000", 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000115392 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.050 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588432+0000", 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6224000000000003e-05 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:59981 10.8 10:11e1be94:::benchmark_data_smithi082_223129_object59980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:54:25.303162+0000", 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 190.686532683, 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.28537527600000001, 2023-10-09T04:57:36.051 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 59981 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303162+0000", 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.052 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303162+0000", 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303161+0000", 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.053 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303184+0000", 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.266e-05 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303190+0000", 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.178e-06 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.054 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.303194+0000", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.5159999999999999e-06 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.305847+0000", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0026535220000000002 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.305982+0000", 2023-10-09T04:57:36.055 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000134912 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.306116+0000", 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013358200000000001 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.308400+0000", 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0022839739999999998 2023-10-09T04:57:36.056 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.309157+0000", 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00075705399999999995 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588468+0000", 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.27931173100000001 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.057 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588505+0000", 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.6640000000000002e-05 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:54:25.588537+0000", 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.1888999999999999e-05 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.058 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169130 10.4 10:2b9607df:::benchmark_data_smithi082_223129_object63681:head [delete] snapc 0=[] ondisk+write+known_if_redirected e311)", 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:45.327390+0000", 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 50.662304716999998, 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.4585058389999999, 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169130 2023-10-09T04:57:36.059 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.060 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.060 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.060 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.060 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327390+0000", 2023-10-09T04:57:36.060 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.060 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.061 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.061 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.061 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327390+0000", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327389+0000", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.999999 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327393+0000", 2023-10-09T04:57:36.062 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.6330000000000004e-06 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327406+0000", 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.2945999999999999e-05 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.327415+0000", 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1980000000000007e-06 2023-10-09T04:57:36.063 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.333208+0000", 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0057924090000000001 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.333390+0000", 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00018227799999999999 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.064 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.333526+0000", 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000136267 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.351007+0000", 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.017480869 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.065 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:45.351021+0000", 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3325e-05 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.335959+0000", 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.98493894299999996 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.335969+0000", 2023-10-09T04:57:36.066 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1419999999999999e-06 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.761744+0000", 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.425775013 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for missing object", 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.761884+0000", 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014030800000000001 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.067 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.785810+0000", 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.023925889999999998 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.785896+0000", 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.5706999999999998e-05 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.068 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169119 10.1 10:8734d847:::benchmark_data_smithi082_223129_object63670:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336327+0000", 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653367736999996, 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.430697901, 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "reached pg", 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.069 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169119 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336327+0000", 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336327+0000", 2023-10-09T04:57:36.070 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336325+0000", 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336332+0000", 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.5599999999999996e-06 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.071 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336346+0000", 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3217000000000001e-05 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336355+0000", 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.414e-06 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.072 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336410+0000", 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5342e-05 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336420+0000", 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.8870000000000004e-06 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.759071+0000", 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.422650685 2023-10-09T04:57:36.073 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.767025+0000", 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0079536829999999996 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169132 10.2 10:47617a5a:::benchmark_data_smithi082_223129_object63683:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.074 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336372+0000", 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653321982000001, 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.406052313, 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169132 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.075 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336372+0000", 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336372+0000", 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.076 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336372+0000", 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336375+0000", 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9840000000000001e-06 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336390+0000", 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4925e-05 2023-10-09T04:57:36.077 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336396+0000", 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.1279999999999996e-06 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336441+0000", 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.4642999999999999e-05 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.078 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336450+0000", 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5489999999999995e-06 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.723312+0000", 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3868621919999999 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.079 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.723536+0000", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000223573 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.723666+0000", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000129925 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.729560+0000", 2023-10-09T04:57:36.080 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0058943540000000001 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.733366+0000", 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0038057529999999998 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.733449+0000", 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.2606000000000002e-05 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.081 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742252+0000", 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0088030529999999999 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742295+0000", 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.3158999999999998e-05 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.082 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742425+0000", 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013000800000000001 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169133 10.1 10:8b8d9b0c:::benchmark_data_smithi082_223129_object63684:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336406+0000", 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653288216, 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.458161155, 2023-10-09T04:57:36.083 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169133 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336406+0000", 2023-10-09T04:57:36.084 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336406+0000", 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336406+0000", 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.085 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336408+0000", 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.6460000000000002e-06 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336415+0000", 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5039999999999999e-06 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.086 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336421+0000", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.6799999999999998e-06 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336450+0000", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9952000000000001e-05 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336458+0000", 2023-10-09T04:57:36.087 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.1239999999999999e-06 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.767433+0000", 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.430975243 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.768525+0000", 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.001092193 2023-10-09T04:57:36.088 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.768667+0000", 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014218 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.779552+0000", 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.010885195 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.089 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.779589+0000", 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.7049999999999999e-05 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.784707+0000", 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0051170319999999997 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.090 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794301+0000", 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0095946390000000003 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794359+0000", 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7714999999999999e-05 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794567+0000", 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00020846800000000001 2023-10-09T04:57:36.091 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169134 10.2 10:4e40cebe:::benchmark_data_smithi082_223129_object63685:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336428+0000", 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653266356000003, 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.406336641, 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.092 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169134 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336428+0000", 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.093 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336428+0000", 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336428+0000", 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.094 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336430+0000", 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.7520000000000001e-06 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336437+0000", 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.2330000000000003e-06 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336442+0000", 2023-10-09T04:57:36.095 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5679999999999999e-06 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336472+0000", 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0205e-05 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336478+0000", 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0730000000000001e-06 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.096 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.729609+0000", 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.3931307259999999 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.729727+0000", 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011767700000000001 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.097 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.729841+0000", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000114252 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.738547+0000", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0087053860000000007 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742480+0000", 2023-10-09T04:57:36.098 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039330119999999996 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742510+0000", 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0947e-05 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742594+0000", 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3640999999999994e-05 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.099 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742648+0000", 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.3908e-05 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742765+0000", 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000116636 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.100 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169137 10.1 10:8c930904:::benchmark_data_smithi082_223129_object63688:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336449+0000", 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653245851000001, 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4583655659999999, 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169137 2023-10-09T04:57:36.101 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336449+0000", 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336449+0000", 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.102 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336448+0000", 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336451+0000", 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.458e-06 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.103 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336459+0000", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.9340000000000006e-06 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336465+0000", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.0270000000000001e-06 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336506+0000", 2023-10-09T04:57:36.104 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1400999999999999e-05 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336516+0000", 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6430000000000007e-06 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.773907+0000", 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.437391458 2023-10-09T04:57:36.105 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.774066+0000", 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015880700000000001 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.774326+0000", 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000260184 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.106 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.779664+0000", 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0053379830000000001 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.784303+0000", 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0046391940000000001 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.107 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.784950+0000", 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000647117 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794623+0000", 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0096726199999999998 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794679+0000", 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.5673e-05 2023-10-09T04:57:36.108 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.794814+0000", 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013548600000000001 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169141 10.2 10:427fbf40:::benchmark_data_smithi082_223129_object63692:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336472+0000", 2023-10-09T04:57:36.109 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653222683000003, 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.4065596629999999, 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169141 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.110 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336472+0000", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336472+0000", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336471+0000", 2023-10-09T04:57:36.111 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336474+0000", 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5409999999999999e-06 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336480+0000", 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.613e-06 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.112 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336486+0000", 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9050000000000002e-06 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336512+0000", 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.5783999999999999e-05 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.113 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336516+0000", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.9840000000000003e-06 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.733746+0000", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.397229893 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.733890+0000", 2023-10-09T04:57:36.114 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00014371100000000001 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.734024+0000", 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000134194 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742555+0000", 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0085314510000000007 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.115 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742853+0000", 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00029799200000000002 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742870+0000", 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.6807999999999999e-05 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.116 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742901+0000", 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.0800000000000003e-05 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.742936+0000", 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.4662000000000001e-05 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743031+0000", 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.5718999999999996e-05 2023-10-09T04:57:36.117 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169146 10.2 10:45f71e61:::benchmark_data_smithi082_223129_object63697:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336494+0000", 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653200145, 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.420542443, 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.118 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169146 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336494+0000", 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.119 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336494+0000", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336494+0000", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336497+0000", 2023-10-09T04:57:36.120 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.9720000000000001e-06 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336503+0000", 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 6.5390000000000004e-06 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336509+0000", 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.733e-06 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.121 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336533+0000", 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.3530999999999999e-05 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336537+0000", 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.1840000000000001e-06 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.122 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.738580+0000", 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.402042995 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.738690+0000", 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.000110398 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:57:36.123 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.738819+0000", 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00012919599999999999 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743101+0000", 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0042814239999999998 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743111+0000", 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.0506000000000001e-05 2023-10-09T04:57:36.124 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.756715+0000", 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.01360374 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.756827+0000", 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00011189 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.125 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.756897+0000", 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 7.0578000000000001e-05 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.757037+0000", 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00013923900000000001 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.126 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "description": "osd_op(client.45806.0:169149 10.2 10:4c5fb476:::benchmark_data_smithi082_223129_object63700:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e313)", 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "initiated_at": "2023-10-09T04:56:46.336517+0000", 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "age": 49.653177569, 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.424783871, 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "type_data": { 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "client_info": { 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "client": "client.45806", 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "client_addr": "172.21.15.82:0/1153783229", 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "tid": 169149 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.127 INFO:teuthology.orchestra.run.smithi082.stdout: "events": [ 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "initiated", 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336517+0000", 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "throttled", 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336517+0000", 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.128 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "header_read", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336516+0000", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "all_read", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336519+0000", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 2.475e-06 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "dispatched", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336527+0000", 2023-10-09T04:57:36.129 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 8.3059999999999998e-06 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "queued_for_pg", 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336533+0000", 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.7320000000000003e-06 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336551+0000", 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.7904999999999999e-05 2023-10-09T04:57:36.130 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for peered", 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:46.336555+0000", 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 3.8449999999999996e-06 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "reached_pg", 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743144+0000", 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 1.406588873 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.131 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "started", 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743240+0000", 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.6732999999999997e-05 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.743331+0000", 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 9.1141e-05 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.132 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.756773+0000", 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.013441944000000001 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.757102+0000", 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00032836399999999999 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.757161+0000", 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 5.9330999999999998e-05 2023-10-09T04:57:36.133 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "op_commit", 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.761108+0000", 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.0039470210000000002 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "commit_sent", 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.761151+0000", 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 4.2497000000000003e-05 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2023-10-09T04:57:36.134 INFO:teuthology.orchestra.run.smithi082.stdout: { 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: "event": "done", 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: "time": "2023-10-09T04:56:47.761301+0000", 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: "duration": 0.00015008400000000001 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: } 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2023-10-09T04:57:36.135 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:57:36.160 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:36.266 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:36.297 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:36.404 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:36.413 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:36 smithi153 bash[59565]: cluster 2023-10-09T04:57:34.916689+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 948 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 6907/45348 objects misplaced (15.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:36.438 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:36.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:36.549 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:36 smithi082 bash[79733]: cluster 2023-10-09T04:57:34.916689+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 948 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 6907/45348 objects misplaced (15.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:36.550 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:36 smithi082 bash[77063]: cluster 2023-10-09T04:57:34.916689+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 948 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 6907/45348 objects misplaced (15.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:36.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:36.688 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.737 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:36.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.871 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:36.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:36.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:37.021 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:37.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:37.126 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:37.264 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.288 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.291 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:37 smithi153 bash[59565]: cluster 2023-10-09T04:57:36.917334+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 900 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6506/43047 objects misplaced (15.114%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:37.395 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:37.424 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:37.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:37 smithi082 bash[79733]: cluster 2023-10-09T04:57:36.917334+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 900 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6506/43047 objects misplaced (15.114%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:37.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:37 smithi082 bash[77063]: cluster 2023-10-09T04:57:36.917334+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 900 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6506/43047 objects misplaced (15.114%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:57:37.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:37.668 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:37.704 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:37.812 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:37.848 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:37.955 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:37.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:38.091 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.114 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:38.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:38.258 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:38.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:38.363 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.396 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:38.501 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:38.540 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:38.647 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:38.692 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:38.797 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:38.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.833 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:38.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:38.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.971 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:38.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:39.076 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:39.122 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:39.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:39.180 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:57:39.227 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:39.245 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:39.270 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:39.378 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:39.407 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:39.513 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:39.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:39.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:39.650 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:39.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:39.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:39.779 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:39.810 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:39.913 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:39.942 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:39.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:39.978 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:39] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-09T04:57:40.046 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:40.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:40.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:39 smithi153 bash[59565]: cluster 2023-10-09T04:57:38.917841+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 753 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5395/35997 objects misplaced (14.987%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:57:40.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:40.224 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:40.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:39 smithi082 bash[77063]: cluster 2023-10-09T04:57:38.917841+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 753 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5395/35997 objects misplaced (14.987%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:57:40.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:40 smithi082 bash[79733]: cluster 2023-10-09T04:57:38.917841+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 753 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5395/35997 objects misplaced (14.987%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:57:40.327 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:40.359 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:40.468 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:40.614 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.636 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:40.739 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:40.762 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:57:40.762 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:40.762 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 3, 4, 7, 6, 5, 2] 2023-10-09T04:57:40.762 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T04:57:40.762 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T04:57:40.767 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:40.870 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:40.895 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:40.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:40.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:41.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:41.012 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:40 smithi153 systemd[1]: Stopping Ceph osd.7 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:57:41.032 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:41.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:41.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:41.269 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:41.277 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:41 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.009307+0000 mon.a (mon.0) 1883 : cluster 1 osd.7 marked itself down and dead 2023-10-09T04:57:41.278 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:41 smithi153 bash[43840]: debug 2023-10-09T04:57:41.007+0000 7f4ae21b7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T04:57:41.278 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:41 smithi153 bash[43840]: debug 2023-10-09T04:57:41.007+0000 7f4ae21b7700 -1 osd.7 320 *** Got signal Terminated *** 2023-10-09T04:57:41.278 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:41 smithi153 bash[43840]: debug 2023-10-09T04:57:41.007+0000 7f4ae21b7700 -1 osd.7 320 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:57:41.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:41.397 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:41.425 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:41 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.009307+0000 mon.a (mon.0) 1883 : cluster 1 osd.7 marked itself down and dead 2023-10-09T04:57:41.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:41 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.009307+0000 mon.a (mon.0) 1883 : cluster 1 osd.7 marked itself down and dead 2023-10-09T04:57:41.528 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:41.548 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:41.651 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:41.676 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:41.758 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:57:41.779 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:41.785 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:57:41.808 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:41.811 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:57:41.836 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:57:41.862 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:57:41.890 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:57:41.911 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:41.918 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.919 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.920 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:57:41.920 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T04:57:41.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:41.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:42.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.071 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:42.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:42.209 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:40.918423+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 716 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5201/34254 objects misplaced (15.184%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.180037+0000 mon.a (mon.0) 1884 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.196397+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.196591+0000 mon.a (mon.0) 1885 : cluster 0 osdmap e321: 8 total, 7 up, 8 in 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.201758+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.219 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.202652+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.220 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: cluster 2023-10-09T04:57:41.203370+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.220 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:42 smithi153 bash[59565]: audit 2023-10-09T04:57:41.728346+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.82:0/713601073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:42.319 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:42.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:42.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:40.918423+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 716 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5201/34254 objects misplaced (15.184%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.180037+0000 mon.a (mon.0) 1884 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.196397+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.196591+0000 mon.a (mon.0) 1885 : cluster 0 osdmap e321: 8 total, 7 up, 8 in 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.201758+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.468 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.202652+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: cluster 2023-10-09T04:57:41.203370+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[77063]: audit 2023-10-09T04:57:41.728346+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.82:0/713601073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:40.918423+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 716 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 5201/34254 objects misplaced (15.184%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.180037+0000 mon.a (mon.0) 1884 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.196397+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.196591+0000 mon.a (mon.0) 1885 : cluster 0 osdmap e321: 8 total, 7 up, 8 in 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.201758+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.202652+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.469 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: cluster 2023-10-09T04:57:41.203370+0000 osd.4 (osd.4) 264 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:57:42.470 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:42 smithi082 bash[79733]: audit 2023-10-09T04:57:41.728346+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.82:0/713601073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:42.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:42.600 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:42.625 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:42.730 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.757 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:42.861 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:42.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:42.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:43.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:43.030 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:43.134 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:43.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:43.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:43.303 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:43.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:43.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.192626+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.193227+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.193872+0000 osd.0 (osd.0) 8 : cluster 1 10.1 continuing backfill to osd.4 from (304'4400,320'12748] MIN to 320'12748 2023-10-09T04:57:43.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.194864+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.410 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.195086+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.198430+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.198916+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.199864+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e322: 8 total, 7 up, 8 in 2023-10-09T04:57:43.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.203279+0000 osd.6 (osd.6) 199 : cluster 1 10.0 continuing backfill to osd.4 from (304'3700,320'12551] 10:0e474cd2:::benchmark_data_smithi082_223129_object98079:head to 320'12551 2023-10-09T04:57:43.411 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.211303+0000 osd.0 (osd.0) 9 : cluster 1 10.c continuing backfill to osd.4 from (304'3900,320'12491] MIN to 320'12491 2023-10-09T04:57:43.411 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[148211]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T04:57:43.411 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:57:43 smithi153 bash[148331]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T04:57:43.445 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:43.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.581 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:43.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:43.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:43.690 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.192626+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.690 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.193227+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.193872+0000 osd.0 (osd.0) 8 : cluster 1 10.1 continuing backfill to osd.4 from (304'4400,320'12748] MIN to 320'12748 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.194864+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.195086+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.198430+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.198916+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.199864+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e322: 8 total, 7 up, 8 in 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.203279+0000 osd.6 (osd.6) 199 : cluster 1 10.0 continuing backfill to osd.4 from (304'3700,320'12551] 10:0e474cd2:::benchmark_data_smithi082_223129_object98079:head to 320'12551 2023-10-09T04:57:43.691 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.211303+0000 osd.0 (osd.0) 9 : cluster 1 10.c continuing backfill to osd.4 from (304'3900,320'12491] MIN to 320'12491 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.192626+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.193227+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.193872+0000 osd.0 (osd.0) 8 : cluster 1 10.1 continuing backfill to osd.4 from (304'4400,320'12748] MIN to 320'12748 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.194864+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.195086+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.693 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.198430+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.694 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.198916+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T04:57:43.694 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.199864+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e322: 8 total, 7 up, 8 in 2023-10-09T04:57:43.694 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.203279+0000 osd.6 (osd.6) 199 : cluster 1 10.0 continuing backfill to osd.4 from (304'3700,320'12551] 10:0e474cd2:::benchmark_data_smithi082_223129_object98079:head to 320'12551 2023-10-09T04:57:43.694 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:43 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.211303+0000 osd.0 (osd.0) 9 : cluster 1 10.c continuing backfill to osd.4 from (304'3900,320'12491] MIN to 320'12491 2023-10-09T04:57:43.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:43.818 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.854 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:43.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:43.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:43.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:43.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:44.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:44.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:44.232 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:44.253 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:44.358 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:44.367 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:44 smithi082 bash[77063]: cluster 2023-10-09T04:57:42.919067+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 8 stale+active+clean, 1 active+remapped+backfilling, 4 active+undersized+degraded, 5 active+remapped+backfill_wait, 61 active+clean; 647 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1730/30942 objects degraded (5.591%); 3988/30942 objects misplaced (12.889%); 4.3 MiB/s, 66 objects/s recovering 2023-10-09T04:57:44.368 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:44 smithi082 bash[77063]: cluster 2023-10-09T04:57:43.205454+0000 mon.a (mon.0) 1887 : cluster 3 Health check failed: Degraded data redundancy: 1730/30942 objects degraded (5.591%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:44.368 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:44 smithi082 bash[79733]: cluster 2023-10-09T04:57:42.919067+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 8 stale+active+clean, 1 active+remapped+backfilling, 4 active+undersized+degraded, 5 active+remapped+backfill_wait, 61 active+clean; 647 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1730/30942 objects degraded (5.591%); 3988/30942 objects misplaced (12.889%); 4.3 MiB/s, 66 objects/s recovering 2023-10-09T04:57:44.368 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:44 smithi082 bash[79733]: cluster 2023-10-09T04:57:43.205454+0000 mon.a (mon.0) 1887 : cluster 3 Health check failed: Degraded data redundancy: 1730/30942 objects degraded (5.591%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:44.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:57:44.414 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:57:44.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:44.526 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:44.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:44.637 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:44 smithi153 bash[59565]: cluster 2023-10-09T04:57:42.919067+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v1448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 8 stale+active+clean, 1 active+remapped+backfilling, 4 active+undersized+degraded, 5 active+remapped+backfill_wait, 61 active+clean; 647 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 1730/30942 objects degraded (5.591%); 3988/30942 objects misplaced (12.889%); 4.3 MiB/s, 66 objects/s recovering 2023-10-09T04:57:44.638 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:44 smithi153 bash[59565]: cluster 2023-10-09T04:57:43.205454+0000 mon.a (mon.0) 1887 : cluster 3 Health check failed: Degraded data redundancy: 1730/30942 objects degraded (5.591%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:44.657 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:44.760 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:44.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:44.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:44.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:44.933 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:44.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:45.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:45.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:45.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:45.173 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:45.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:45.312 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:45.342 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:45.450 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:45.478 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:45.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:45.592 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:45 smithi153 bash[59565]: audit 2023-10-09T04:57:44.311193+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:45.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:45 smithi153 bash[59565]: audit 2023-10-09T04:57:44.312533+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:45.593 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:45 smithi153 bash[59565]: cluster 2023-10-09T04:57:44.919656+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 5 stale+active+clean, 5 active+undersized+degraded, 5 active+remapped+backfill_wait, 56 active+clean; 594 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1777/28389 objects degraded (6.259%); 3731/28389 objects misplaced (13.142%); 4.8 MiB/s, 72 objects/s recovering 2023-10-09T04:57:45.617 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:45.721 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:45.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[77063]: audit 2023-10-09T04:57:44.311193+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:45.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[77063]: audit 2023-10-09T04:57:44.312533+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:45.729 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[77063]: cluster 2023-10-09T04:57:44.919656+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 5 stale+active+clean, 5 active+undersized+degraded, 5 active+remapped+backfill_wait, 56 active+clean; 594 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1777/28389 objects degraded (6.259%); 3731/28389 objects misplaced (13.142%); 4.8 MiB/s, 72 objects/s recovering 2023-10-09T04:57:45.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[79733]: audit 2023-10-09T04:57:44.311193+0000 mon.a (mon.0) 1888 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:57:45.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[79733]: audit 2023-10-09T04:57:44.312533+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:45.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:45 smithi082 bash[79733]: cluster 2023-10-09T04:57:44.919656+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v1449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 5 stale+active+clean, 5 active+undersized+degraded, 5 active+remapped+backfill_wait, 56 active+clean; 594 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1777/28389 objects degraded (6.259%); 3731/28389 objects misplaced (13.142%); 4.8 MiB/s, 72 objects/s recovering 2023-10-09T04:57:45.747 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:45.856 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:45.889 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:45.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:45.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:45.898 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:45.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:46.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:46.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:46.188 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:46.294 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.328 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:46.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:57:46.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:46.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:46.581 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:57:46.599 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:46.639 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:46.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.774 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:46.876 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.884 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:46 smithi082 bash[77063]: audit 2023-10-09T04:57:46.508673+0000 mon.a (mon.0) 1890 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:46.885 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:46 smithi082 bash[79733]: audit 2023-10-09T04:57:46.508673+0000 mon.a (mon.0) 1890 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:46.895 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:46 smithi153 bash[59565]: audit 2023-10-09T04:57:46.508673+0000 mon.a (mon.0) 1890 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:46.897 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T04:57:46.904 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:46.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:46.924 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:57:46.924 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T04:57:47.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:47.046 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:47.152 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:47.178 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:47.281 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:47.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:47.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:47.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:47.554 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:47.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:47.696 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:47.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: cluster 2023-10-09T04:57:46.559317+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:47.705 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: cluster 2023-10-09T04:57:46.564580+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: audit 2023-10-09T04:57:46.566900+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: cluster 2023-10-09T04:57:46.567033+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e323: 8 total, 7 up, 8 in 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: cluster 2023-10-09T04:57:46.568565+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[77063]: cluster 2023-10-09T04:57:46.920183+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v1451: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 566 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 2281/27012 objects degraded (8.444%); 3578/27012 objects misplaced (13.246%); 5.2 MiB/s, 77 objects/s recovering 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: cluster 2023-10-09T04:57:46.559317+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: cluster 2023-10-09T04:57:46.564580+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.706 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: audit 2023-10-09T04:57:46.566900+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:47.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: cluster 2023-10-09T04:57:46.567033+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e323: 8 total, 7 up, 8 in 2023-10-09T04:57:47.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: cluster 2023-10-09T04:57:46.568565+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.707 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:47 smithi082 bash[79733]: cluster 2023-10-09T04:57:46.920183+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v1451: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 566 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 2281/27012 objects degraded (8.444%); 3578/27012 objects misplaced (13.246%); 5.2 MiB/s, 77 objects/s recovering 2023-10-09T04:57:47.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:47.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:47.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: cluster 2023-10-09T04:57:46.559317+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: cluster 2023-10-09T04:57:46.564580+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: audit 2023-10-09T04:57:46.566900+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.82:0/1705593075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:47.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: cluster 2023-10-09T04:57:46.567033+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e323: 8 total, 7 up, 8 in 2023-10-09T04:57:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: cluster 2023-10-09T04:57:46.568565+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T04:57:47.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:47 smithi153 bash[59565]: cluster 2023-10-09T04:57:46.920183+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v1451: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 566 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 2281/27012 objects degraded (8.444%); 3578/27012 objects misplaced (13.246%); 5.2 MiB/s, 77 objects/s recovering 2023-10-09T04:57:47.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:47.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:47.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:47.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:48.099 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:48.235 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:48.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:48.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:48.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.398 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:48.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:48.548 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:48.656 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:48.692 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:48.795 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.817 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:48.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:48.921 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:48.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:48.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:49.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:49.176 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:49.207 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:49.311 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:49.435 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:49.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:49.692 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:49.718 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:49.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:49.852 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:49.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:49 smithi153 bash[59565]: cluster 2023-10-09T04:57:48.595912+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.853 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:49 smithi153 bash[59565]: cluster 2023-10-09T04:57:48.596164+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e324: 8 total, 7 up, 8 in 2023-10-09T04:57:49.854 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:49 smithi153 bash[59565]: cluster 2023-10-09T04:57:48.604482+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.854 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:49 smithi153 bash[59565]: cluster 2023-10-09T04:57:48.605885+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.854 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:49 smithi153 bash[59565]: cluster 2023-10-09T04:57:48.920827+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v1453: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 408 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1313/19452 objects degraded (6.750%); 2575/19452 objects misplaced (13.238%); 9.1 MiB/s, 140 objects/s recovering 2023-10-09T04:57:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:49.955 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[77063]: cluster 2023-10-09T04:57:48.595912+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[77063]: cluster 2023-10-09T04:57:48.596164+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e324: 8 total, 7 up, 8 in 2023-10-09T04:57:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[77063]: cluster 2023-10-09T04:57:48.604482+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[77063]: cluster 2023-10-09T04:57:48.605885+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[77063]: cluster 2023-10-09T04:57:48.920827+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v1453: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 408 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1313/19452 objects degraded (6.750%); 2575/19452 objects misplaced (13.238%); 9.1 MiB/s, 140 objects/s recovering 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:49] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[79733]: cluster 2023-10-09T04:57:48.595912+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[79733]: cluster 2023-10-09T04:57:48.596164+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e324: 8 total, 7 up, 8 in 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[79733]: cluster 2023-10-09T04:57:48.604482+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[79733]: cluster 2023-10-09T04:57:48.605885+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T04:57:49.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:49 smithi082 bash[79733]: cluster 2023-10-09T04:57:48.920827+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v1453: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 20 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 5 active+remapped+backfill_wait, 49 active+clean; 408 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 1313/19452 objects degraded (6.750%); 2575/19452 objects misplaced (13.238%); 9.1 MiB/s, 140 objects/s recovering 2023-10-09T04:57:49.984 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:49.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:50.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:50.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:50.219 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:50.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:50.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:50.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.366 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:50.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.511 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:50.619 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:50.643 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:50.747 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:50.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:49.592514+0000 mon.a (mon.0) 1895 : cluster 3 Health check update: Degraded data redundancy: 1313/19452 objects degraded (6.750%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:50.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:49.600976+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:49.607943+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e325: 8 total, 7 up, 8 in 2023-10-09T04:57:50.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:49.608605+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:49.611717+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.892 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:50.186823+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e326: 8 total, 7 up, 8 in 2023-10-09T04:57:50.892 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:50 smithi153 bash[59565]: cluster 2023-10-09T04:57:50.187311+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:50.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:50.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:50.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:49.592514+0000 mon.a (mon.0) 1895 : cluster 3 Health check update: Degraded data redundancy: 1313/19452 objects degraded (6.750%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:50.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:49.600976+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:49.607943+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e325: 8 total, 7 up, 8 in 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:49.608605+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:49.611717+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:50.186823+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e326: 8 total, 7 up, 8 in 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[77063]: cluster 2023-10-09T04:57:50.187311+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:49.592514+0000 mon.a (mon.0) 1895 : cluster 3 Health check update: Degraded data redundancy: 1313/19452 objects degraded (6.750%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:49.600976+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:49.607943+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e325: 8 total, 7 up, 8 in 2023-10-09T04:57:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:49.608605+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:49.611717+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T04:57:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:50.186823+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e326: 8 total, 7 up, 8 in 2023-10-09T04:57:50.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:50 smithi082 bash[79733]: cluster 2023-10-09T04:57:50.187311+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.013 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.033 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:57:51.042 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:51.146 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.171 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:51.252 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:51.274 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:51.418 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.448 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:51.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:51.577 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:51.684 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:51.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:51.924 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 6, 5, 2] 2023-10-09T04:57:51.925 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:51.925 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:57:51.925 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:57:51.925 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:57:51.925 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:57:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:51 smithi153 bash[59565]: cluster 2023-10-09T04:57:50.182059+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:51 smithi153 bash[59565]: cluster 2023-10-09T04:57:50.183953+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:51 smithi153 bash[59565]: cluster 2023-10-09T04:57:50.921900+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v1456: 89 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 4 active+remapped+backfill_wait, 49 active+clean; 370 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 746/17625 objects degraded (4.233%); 1895/17625 objects misplaced (10.752%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:57:51.926 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:51 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.200207+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e327: 8 total, 7 up, 8 in 2023-10-09T04:57:51.946 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.954 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[77063]: cluster 2023-10-09T04:57:50.182059+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.954 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[77063]: cluster 2023-10-09T04:57:50.183953+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[77063]: cluster 2023-10-09T04:57:50.921900+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v1456: 89 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 4 active+remapped+backfill_wait, 49 active+clean; 370 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 746/17625 objects degraded (4.233%); 1895/17625 objects misplaced (10.752%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.200207+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e327: 8 total, 7 up, 8 in 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[79733]: cluster 2023-10-09T04:57:50.182059+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[79733]: cluster 2023-10-09T04:57:50.183953+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[79733]: cluster 2023-10-09T04:57:50.921900+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v1456: 89 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 4 active+remapped+backfill_wait, 49 active+clean; 370 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 746/17625 objects degraded (4.233%); 1895/17625 objects misplaced (10.752%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:57:51.955 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:51 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.200207+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e327: 8 total, 7 up, 8 in 2023-10-09T04:57:51.970 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:51.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:52.074 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:52.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:52.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:52.233 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:52.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:52.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:52.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:52.506 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:52.611 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:52.639 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:52.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:52.743 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:52.775 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:52.879 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:52.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.195399+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.887 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.198892+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.202803+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.205594+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:51.297129+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:52.197337+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:52.888 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:52 smithi153 bash[59565]: cluster 2023-10-09T04:57:52.203049+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e328: 8 total, 7 up, 8 in 2023-10-09T04:57:52.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:52.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.195399+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.198892+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.202803+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.205594+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:51.297129+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:52.197337+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:52.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[77063]: cluster 2023-10-09T04:57:52.203049+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e328: 8 total, 7 up, 8 in 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.195399+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.198892+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.202803+0000 osd.4 (osd.4) 273 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.205594+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:51.297129+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:52.197337+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:52.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:52 smithi082 bash[79733]: cluster 2023-10-09T04:57:52.203049+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e328: 8 total, 7 up, 8 in 2023-10-09T04:57:53.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:53.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:53.140 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:53.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:53.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.296 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:53.403 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.430 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:53.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:53.553 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:53.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.686 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:53.790 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:53.797 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[79733]: cluster 2023-10-09T04:57:52.197553+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.797 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[79733]: cluster 2023-10-09T04:57:52.203630+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.797 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[79733]: cluster 2023-10-09T04:57:52.922604+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v1459: 89 pgs: 1 remapped+peering, 2 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 3 active+remapped+backfill_wait, 49 active+clean; 334 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 629/15894 objects degraded (3.957%); 1319/15894 objects misplaced (8.299%); 12 MiB/s, 184 objects/s recovering 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[79733]: cluster 2023-10-09T04:57:53.205444+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e329: 8 total, 7 up, 8 in 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[79733]: cluster 2023-10-09T04:57:53.206934+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[77063]: cluster 2023-10-09T04:57:52.197553+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[77063]: cluster 2023-10-09T04:57:52.203630+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[77063]: cluster 2023-10-09T04:57:52.922604+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v1459: 89 pgs: 1 remapped+peering, 2 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 3 active+remapped+backfill_wait, 49 active+clean; 334 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 629/15894 objects degraded (3.957%); 1319/15894 objects misplaced (8.299%); 12 MiB/s, 184 objects/s recovering 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[77063]: cluster 2023-10-09T04:57:53.205444+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e329: 8 total, 7 up, 8 in 2023-10-09T04:57:53.798 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:53 smithi082 bash[77063]: cluster 2023-10-09T04:57:53.206934+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:53.813 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:53.897 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:57:53.897 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:57:53.915 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:53 smithi153 bash[59565]: cluster 2023-10-09T04:57:52.197553+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:53 smithi153 bash[59565]: cluster 2023-10-09T04:57:52.203630+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T04:57:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:53 smithi153 bash[59565]: cluster 2023-10-09T04:57:52.922604+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v1459: 89 pgs: 1 remapped+peering, 2 active+clean+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 3 active+remapped+backfill_wait, 49 active+clean; 334 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 296 op/s; 629/15894 objects degraded (3.957%); 1319/15894 objects misplaced (8.299%); 12 MiB/s, 184 objects/s recovering 2023-10-09T04:57:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:53 smithi153 bash[59565]: cluster 2023-10-09T04:57:53.205444+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e329: 8 total, 7 up, 8 in 2023-10-09T04:57:53.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:53 smithi153 bash[59565]: cluster 2023-10-09T04:57:53.206934+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:53.937 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:53.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:54.041 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:54.070 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:54.174 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:54.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:54.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.336 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:54.444 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:54.465 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:54.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:54.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.592 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:54.695 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.722 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:54.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:54.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[77063]: cluster 2023-10-09T04:57:53.202746+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.838 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[77063]: cluster 2023-10-09T04:57:53.214523+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.207017+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e330: 8 total, 7 up, 8 in 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.208506+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.211318+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[79733]: cluster 2023-10-09T04:57:53.202746+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[79733]: cluster 2023-10-09T04:57:53.214523+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.207017+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e330: 8 total, 7 up, 8 in 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.208506+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.839 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:54 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.211318+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.858 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:54.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:54 smithi153 bash[59565]: cluster 2023-10-09T04:57:53.202746+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:54 smithi153 bash[59565]: cluster 2023-10-09T04:57:53.214523+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T04:57:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:54 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.207017+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e330: 8 total, 7 up, 8 in 2023-10-09T04:57:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:54 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.208506+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:54 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.211318+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:54.961 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:54.992 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:54.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:55.096 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:55.126 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:55.229 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:55.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:55.361 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:55.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:55.492 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:55.515 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:55.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:55.618 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:55.642 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:55.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:55.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:55.774 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:55.878 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.205252+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.206314+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.206576+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:54.923585+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v1462: 89 pgs: 1 remapped+peering, 3 active+clean+remapped, 1 active+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 50 active+clean; 294 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 628/13995 objects degraded (4.487%); 980/13995 objects misplaced (7.003%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.180829+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: Degraded data redundancy: 629/15894 objects degraded (3.957%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.191722+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e331: 8 total, 7 up, 8 in 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.192334+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:55.891 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:55 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.194470+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:55.910 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:55.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:55.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.205252+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.206314+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.206576+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:54.923585+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v1462: 89 pgs: 1 remapped+peering, 3 active+clean+remapped, 1 active+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 50 active+clean; 294 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 628/13995 objects degraded (4.487%); 980/13995 objects misplaced (7.003%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.180829+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: Degraded data redundancy: 629/15894 objects degraded (3.957%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:55.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.191722+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e331: 8 total, 7 up, 8 in 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.192334+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.194470+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.205252+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.206314+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.206576+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:54.923585+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v1462: 89 pgs: 1 remapped+peering, 3 active+clean+remapped, 1 active+remapped+backfilling, 20 active+undersized, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 50 active+clean; 294 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 628/13995 objects degraded (4.487%); 980/13995 objects misplaced (7.003%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T04:57:55.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.180829+0000 mon.a (mon.0) 1902 : cluster 3 Health check update: Degraded data redundancy: 629/15894 objects degraded (3.957%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:57:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.191722+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e331: 8 total, 7 up, 8 in 2023-10-09T04:57:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.192334+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:55.983 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:55 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.194470+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.014 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.044 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:56.148 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:56.185 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:56.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:57:56.290 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:56.317 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:56.422 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.444 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:56.552 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:56.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:56.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:56.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.185471+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.192402+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[77063]: cluster 2023-10-09T04:57:55.194994+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[77063]: cluster 2023-10-09T04:57:56.203174+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e332: 8 total, 7 up, 8 in 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.185471+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.192402+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[79733]: cluster 2023-10-09T04:57:55.194994+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:56 smithi082 bash[79733]: cluster 2023-10-09T04:57:56.203174+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e332: 8 total, 7 up, 8 in 2023-10-09T04:57:56.710 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:57:56.718 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:56.720 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:56.823 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.852 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:56 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.185471+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:56 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.192402+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:56 smithi153 bash[59565]: cluster 2023-10-09T04:57:55.194994+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T04:57:56.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:56 smithi153 bash[59565]: cluster 2023-10-09T04:57:56.203174+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e332: 8 total, 7 up, 8 in 2023-10-09T04:57:56.960 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:56.985 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:56.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:57.090 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.113 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:57.216 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:57.257 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:57.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.387 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:57.492 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:57.517 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:57.623 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:57.651 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:57.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.778 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:57.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:57.902 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[77063]: cluster 2023-10-09T04:57:56.199126+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[77063]: cluster 2023-10-09T04:57:56.202998+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[77063]: cluster 2023-10-09T04:57:56.205675+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[77063]: cluster 2023-10-09T04:57:56.924170+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v1465: 89 pgs: 1 peering, 4 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 52 active+clean; 260 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 522/12345 objects degraded (4.228%); 465/12345 objects misplaced (3.767%); 14 MiB/s, 220 objects/s recovering 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[79733]: cluster 2023-10-09T04:57:56.199126+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[79733]: cluster 2023-10-09T04:57:56.202998+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[79733]: cluster 2023-10-09T04:57:56.205675+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.904 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:57 smithi082 bash[79733]: cluster 2023-10-09T04:57:56.924170+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v1465: 89 pgs: 1 peering, 4 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 52 active+clean; 260 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 522/12345 objects degraded (4.228%); 465/12345 objects misplaced (3.767%); 14 MiB/s, 220 objects/s recovering 2023-10-09T04:57:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:57.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:57.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:57 smithi153 bash[59565]: cluster 2023-10-09T04:57:56.199126+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:57 smithi153 bash[59565]: cluster 2023-10-09T04:57:56.202998+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:57 smithi153 bash[59565]: cluster 2023-10-09T04:57:56.205675+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T04:57:57.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:57 smithi153 bash[59565]: cluster 2023-10-09T04:57:56.924170+0000 mgr.y (mgr.34107) 1321 : cluster 0 pgmap v1465: 89 pgs: 1 peering, 4 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 52 active+clean; 260 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 522/12345 objects degraded (4.228%); 465/12345 objects misplaced (3.767%); 14 MiB/s, 220 objects/s recovering 2023-10-09T04:57:58.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:58.044 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:58.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:58.170 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:58.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:58.273 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:58.300 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:58.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:58.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:58.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:58.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:58.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:58.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:58.699 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:58.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:58.860 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:58.966 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:58.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:58.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:59.043 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:57:59.077 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:57:59.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.125 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:57:59.126 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:59.158 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:57:59.194 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:57:59.230 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.243 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:57:59.259 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:59.276 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:57:59.278 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:57:59.279 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T04:57:59.296 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:57:59 smithi082 bash[77063]: audit 2023-10-09T04:57:58.969978+0000 mon.a (mon.0) 1905 : audit 0 from='client.? 172.21.15.82:0/2021131110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:59.296 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:57:59 smithi082 bash[79733]: audit 2023-10-09T04:57:58.969978+0000 mon.a (mon.0) 1905 : audit 0 from='client.? 172.21.15.82:0/2021131110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:59.362 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:59.389 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:59.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:57:59 smithi153 bash[59565]: audit 2023-10-09T04:57:58.969978+0000 mon.a (mon.0) 1905 : audit 0 from='client.? 172.21.15.82:0/2021131110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:59.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:59.519 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:59.624 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:59.647 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:59.754 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:59.792 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:59.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:59.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:57:59.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:57:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:57:59] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T04:58:00.025 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.054 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:00.157 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:00.187 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:00.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:00.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:00 smithi082 bash[77063]: cluster 2023-10-09T04:57:58.924874+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 54 active+clean; 186 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 429/8781 objects degraded (4.886%); 465/8781 objects misplaced (5.296%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:58:00.306 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:00 smithi082 bash[77063]: audit 2023-10-09T04:57:59.307241+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:00.306 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:00 smithi082 bash[79733]: cluster 2023-10-09T04:57:58.924874+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 54 active+clean; 186 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 429/8781 objects degraded (4.886%); 465/8781 objects misplaced (5.296%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:58:00.306 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:00 smithi082 bash[79733]: audit 2023-10-09T04:57:59.307241+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:00.320 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:00.424 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:00 smithi153 bash[59565]: cluster 2023-10-09T04:57:58.924874+0000 mgr.y (mgr.34107) 1322 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 54 active+clean; 186 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 429/8781 objects degraded (4.886%); 465/8781 objects misplaced (5.296%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:58:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:00 smithi153 bash[59565]: audit 2023-10-09T04:57:59.307241+0000 mon.a (mon.0) 1906 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:00.454 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:00.563 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:00.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:00.691 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.718 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:00.824 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:00.855 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:00.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.993 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:00.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:01.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:01.126 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:01.230 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:01.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:01 smithi082 bash[77063]: cluster 2023-10-09T04:58:00.183290+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 429/8781 objects degraded (4.886%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:01.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:01 smithi082 bash[79733]: cluster 2023-10-09T04:58:00.183290+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 429/8781 objects degraded (4.886%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:01.239 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:01 smithi153 bash[59565]: cluster 2023-10-09T04:58:00.183290+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: Degraded data redundancy: 429/8781 objects degraded (4.886%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:01.268 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:01.372 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:01.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:01.494 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Removed 105446 objects 2023-10-09T04:58:01.494 INFO:tasks.radosbench.radosbench.0.smithi082.stdout:Clean up completed and total clean up time :165.077 2023-10-09T04:58:01.515 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:01.538 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:01.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:01.641 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:01.660 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:01.762 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:01.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:01.887 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:01.915 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:01.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:02.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:02.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:02.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:02.151 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:02.186 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:02.289 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:02.323 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:02.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:02 smithi153 bash[59565]: cluster 2023-10-09T04:58:00.925644+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v1467: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 104 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 260/4860 objects degraded (5.350%); 100/4860 objects misplaced (2.058%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:58:02.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.454 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.457 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:02 smithi082 bash[77063]: cluster 2023-10-09T04:58:00.925644+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v1467: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 104 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 260/4860 objects degraded (5.350%); 100/4860 objects misplaced (2.058%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:58:02.458 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:02 smithi082 bash[79733]: cluster 2023-10-09T04:58:00.925644+0000 mgr.y (mgr.34107) 1323 : cluster 0 pgmap v1467: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 104 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 260/4860 objects degraded (5.350%); 100/4860 objects misplaced (2.058%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:58:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:58:02.477 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:58:02.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:02.584 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:02.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:02.729 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:58:02.730 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "" 2023-10-09T04:58:02.730 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:58:02.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.846 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:02.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:02.950 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:02.963 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:02.975 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:02.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:02.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:03.060 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T04:58:03.060 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T04:58:03.080 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:03.130 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:03.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:03.265 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:03.368 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:03.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:03.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:03.525 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:03.629 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:03.655 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:03.757 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:03.789 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:03.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:03.921 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:03.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:04.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:04.056 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:04.161 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.183 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:04.285 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:04.311 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:04.415 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:04 smithi153 bash[59565]: cluster 2023-10-09T04:58:02.926188+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v1468: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 71 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 73/3258 objects degraded (2.241%); 100/3258 objects misplaced (3.069%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:58:04.427 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:04 smithi082 bash[77063]: cluster 2023-10-09T04:58:02.926188+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v1468: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 71 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 73/3258 objects degraded (2.241%); 100/3258 objects misplaced (3.069%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:58:04.427 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:04 smithi082 bash[79733]: cluster 2023-10-09T04:58:02.926188+0000 mgr.y (mgr.34107) 1324 : cluster 0 pgmap v1468: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 71 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 73/3258 objects degraded (2.241%); 100/3258 objects misplaced (3.069%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:58:04.446 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:04.478 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:58:04.549 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:04.584 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:04.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:04.713 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:04.816 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:04.841 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:04.944 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:04.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:05.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:05.101 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:05.205 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:05.207 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:58:05.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:05 smithi082 bash[79733]: audit 2023-10-09T04:58:04.827403+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:05.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:05 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.185376+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 73/3258 objects degraded (2.241%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:05 smithi082 bash[77063]: audit 2023-10-09T04:58:04.827403+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:05.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:05 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.185376+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 73/3258 objects degraded (2.241%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:05.233 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:05.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:05.363 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:05.466 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:05.494 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:05.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:05.596 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:05.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:05 smithi153 bash[59565]: audit 2023-10-09T04:58:04.827403+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:05.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:05 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.185376+0000 mon.a (mon.0) 1909 : cluster 3 Health check update: Degraded data redundancy: 73/3258 objects degraded (2.241%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:05.732 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:05.763 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:05.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:05.902 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:05.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:06.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:06.038 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:06.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:06.172 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:06.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:06.305 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:06.408 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:06 smithi153 bash[59565]: cluster 2023-10-09T04:58:04.927122+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v1469: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 29 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 73/1272 objects degraded (5.739%); 100/1272 objects misplaced (7.862%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:58:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:06 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.193662+0000 mon.a (mon.0) 1910 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:06 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.199275+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:06 smithi153 bash[59565]: audit 2023-10-09T04:58:05.202801+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:06.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:06 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.202885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e333: 8 total, 7 up, 8 in 2023-10-09T04:58:06.431 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[77063]: cluster 2023-10-09T04:58:04.927122+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v1469: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 29 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 73/1272 objects degraded (5.739%); 100/1272 objects misplaced (7.862%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:58:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.193662+0000 mon.a (mon.0) 1910 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.199275+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[77063]: audit 2023-10-09T04:58:05.202801+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.202885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e333: 8 total, 7 up, 8 in 2023-10-09T04:58:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[79733]: cluster 2023-10-09T04:58:04.927122+0000 mgr.y (mgr.34107) 1325 : cluster 0 pgmap v1469: 89 pgs: 2 active+clean+remapped, 20 active+undersized, 12 active+undersized+degraded, 55 active+clean; 29 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 73/1272 objects degraded (5.739%); 100/1272 objects misplaced (7.862%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:58:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.193662+0000 mon.a (mon.0) 1910 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.199275+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[79733]: audit 2023-10-09T04:58:05.202801+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.82:0/508243671' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:06.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:06 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.202885+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e333: 8 total, 7 up, 8 in 2023-10-09T04:58:06.536 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:06.573 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:06.677 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:06.703 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:06.805 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:06.836 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:06.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:06.969 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:06.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:07.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:07.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:07.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.440704+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.443568+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.230 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[77063]: cluster 2023-10-09T04:58:05.733221+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.230 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.440704+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.443568+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:07 smithi082 bash[79733]: cluster 2023-10-09T04:58:05.733221+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.232 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:07.334 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:07.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:07.471 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:07.503 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:07.608 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:07.620 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:07 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.440704+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.620 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:07 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.443568+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.620 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:07 smithi153 bash[59565]: cluster 2023-10-09T04:58:05.733221+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T04:58:07.641 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:07.745 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:07.771 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:07.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:07.897 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:07.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:08.001 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.026 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:08.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:08.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:08.294 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:08.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.406 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:08 smithi082 bash[77063]: cluster 2023-10-09T04:58:06.928101+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v1471: 89 pgs: 2 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 17/84 objects degraded (20.238%); 0 B/s, 0 objects/s recovering 2023-10-09T04:58:08.406 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:08 smithi082 bash[79733]: cluster 2023-10-09T04:58:06.928101+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v1471: 89 pgs: 2 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 17/84 objects degraded (20.238%); 0 B/s, 0 objects/s recovering 2023-10-09T04:58:08.426 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:08.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:08.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:08.623 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 6, 5, 2] 2023-10-09T04:58:08.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:58:08.660 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:08.670 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:08 smithi153 bash[59565]: cluster 2023-10-09T04:58:06.928101+0000 mgr.y (mgr.34107) 1326 : cluster 0 pgmap v1471: 89 pgs: 2 active+clean+remapped, 21 active+undersized, 11 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 17/84 objects degraded (20.238%); 0 B/s, 0 objects/s recovering 2023-10-09T04:58:08.687 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:08.732 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:08.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:08.819 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:08.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:08.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:08.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:08.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:09.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:09.087 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:09.156 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:58:09.156 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:58:09.157 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:58:09.157 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:58:09.157 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:58:09.191 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:09.225 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:09.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:09.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:58:09.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:09.355 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:09.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:09 smithi153 bash[59565]: audit 2023-10-09T04:58:08.351895+0000 mon.a (mon.0) 1913 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:09.458 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:09 smithi082 bash[77063]: audit 2023-10-09T04:58:08.351895+0000 mon.a (mon.0) 1913 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:09.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:09 smithi082 bash[79733]: audit 2023-10-09T04:58:08.351895+0000 mon.a (mon.0) 1913 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:09.488 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:09.592 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:09.726 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:09.752 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:09.855 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.890 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:09.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:09.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:09] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T04:58:09.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:10.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:10.127 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:10.161 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:10.265 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:10.301 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:10.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:10.413 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:10 smithi082 bash[77063]: cluster 2023-10-09T04:58:08.928838+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v1472: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:10.414 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:10 smithi082 bash[77063]: cluster 2023-10-09T04:58:10.186946+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:10.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:10 smithi082 bash[79733]: cluster 2023-10-09T04:58:08.928838+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v1472: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:10.414 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:10 smithi082 bash[79733]: cluster 2023-10-09T04:58:10.186946+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:10 smithi153 bash[59565]: cluster 2023-10-09T04:58:08.928838+0000 mgr.y (mgr.34107) 1327 : cluster 0 pgmap v1472: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:10 smithi153 bash[59565]: cluster 2023-10-09T04:58:10.186946+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:10.432 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:10.535 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:10.567 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:10.670 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:10.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:10.800 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:10.832 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:10.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:10.935 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:10.956 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:10.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:11.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.085 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:11.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:11.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:11.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:11.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:11.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:58:11.453 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.490 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:11.593 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:11.624 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:11.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.761 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:11.865 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.887 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:11.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:11.890 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:11.991 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:12.013 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:12.117 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:12.150 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:12.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:12.254 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:12.286 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:12.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:12.390 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:12.398 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:12 smithi082 bash[77063]: cluster 2023-10-09T04:58:10.929687+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v1473: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:12.398 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:12 smithi082 bash[79733]: cluster 2023-10-09T04:58:10.929687+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v1473: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:12.415 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:12.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:12 smithi153 bash[59565]: cluster 2023-10-09T04:58:10.929687+0000 mgr.y (mgr.34107) 1328 : cluster 0 pgmap v1473: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:12.519 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.555 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:12.659 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:12.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:12.798 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.827 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:12.931 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.960 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:12.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:13.064 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.093 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:13.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:13.223 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:13.328 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:13.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.527 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:13.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:13.660 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:13.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:13.789 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:13.805 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:13.892 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:13.943 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:13.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:14.046 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:14.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:14.177 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:14.204 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:14.307 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:14.330 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:14.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:14 smithi153 bash[59565]: cluster 2023-10-09T04:58:12.930370+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v1474: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:14.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:14.480 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:14.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:14 smithi082 bash[77063]: cluster 2023-10-09T04:58:12.930370+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v1474: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:14.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:14 smithi082 bash[79733]: cluster 2023-10-09T04:58:12.930370+0000 mgr.y (mgr.34107) 1329 : cluster 0 pgmap v1474: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:14.584 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:14.612 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:14.715 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:14.740 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:14.843 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:14.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:14.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:14.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:14.976 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:14.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:15.101 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:15.132 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:15.236 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.254 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:15.357 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:15.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.312137+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.313737+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.640422+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.646186+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.894053+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: audit 2023-10-09T04:58:14.899165+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.426 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:15 smithi153 bash[59565]: cluster 2023-10-09T04:58:14.930852+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v1475: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 39 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:15.482 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.312137+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.313737+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.640422+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.646186+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.894053+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: audit 2023-10-09T04:58:14.899165+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[77063]: cluster 2023-10-09T04:58:14.930852+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v1475: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 39 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.312137+0000 mon.a (mon.0) 1915 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.313737+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.640422+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.646186+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.894053+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: audit 2023-10-09T04:58:14.899165+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:15.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:15 smithi082 bash[79733]: cluster 2023-10-09T04:58:14.930852+0000 mgr.y (mgr.34107) 1330 : cluster 0 pgmap v1475: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 39 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:15.615 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:15.646 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:15.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:15.751 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:15.851 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:15.880 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:15.907 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:16.010 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:16.042 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:16.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:16.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:16.272 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:16.306 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:16.410 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:16.438 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[79733]: audit 2023-10-09T04:58:15.321998+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[79733]: audit 2023-10-09T04:58:15.323210+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[79733]: audit 2023-10-09T04:58:15.330394+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[77063]: audit 2023-10-09T04:58:15.321998+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[77063]: audit 2023-10-09T04:58:15.323210+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:16.485 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:16 smithi082 bash[77063]: audit 2023-10-09T04:58:15.330394+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:16.541 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:16.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:16.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:16 smithi153 bash[59565]: audit 2023-10-09T04:58:15.321998+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:16 smithi153 bash[59565]: audit 2023-10-09T04:58:15.323210+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:16 smithi153 bash[59565]: audit 2023-10-09T04:58:15.330394+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:16.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:16.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:16.835 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:16.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:16.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:16.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:17.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:17.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:17.198 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:17.219 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:17.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.349 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:17.452 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.481 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:17.526 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:17 smithi082 bash[77063]: cluster 2023-10-09T04:58:16.931432+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v1476: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:17.527 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:17 smithi082 bash[79733]: cluster 2023-10-09T04:58:16.931432+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v1476: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:17.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:17.599 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:17 smithi153 bash[59565]: cluster 2023-10-09T04:58:16.931432+0000 mgr.y (mgr.34107) 1331 : cluster 0 pgmap v1476: 89 pgs: 2 active+clean+remapped, 22 active+undersized, 10 active+undersized+degraded, 55 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:17.620 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:17.723 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.749 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:17.852 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:17.883 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:17.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:17.986 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.005 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:18.109 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.132 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:18.234 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:18.253 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:18.356 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:18.362 INFO:teuthology.orchestra.run.smithi082.stderr:pool 'unique_pool_5' removed 2023-10-09T04:58:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:18.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:18.493 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:18.522 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:18.627 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:18.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:18 smithi153 bash[59565]: audit 2023-10-09T04:58:18.166320+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:58:18.653 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:18.756 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:18.787 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:18.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:18.881 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:18 smithi082 bash[77063]: audit 2023-10-09T04:58:18.166320+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:58:18.881 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:18 smithi082 bash[79733]: audit 2023-10-09T04:58:18.166320+0000 mon.a (mon.0) 1924 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:58:18.891 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.923 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:18.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:19.026 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:19.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:19.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:19.202 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:19.305 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.338 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:19.441 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:19.445 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[77063]: cluster 2023-10-09T04:58:18.329812+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[77063]: audit 2023-10-09T04:58:18.331868+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[77063]: cluster 2023-10-09T04:58:18.331950+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e334: 8 total, 7 up, 8 in 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[77063]: cluster 2023-10-09T04:58:18.932007+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v1478: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[77063]: cluster 2023-10-09T04:58:19.047773+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[79733]: cluster 2023-10-09T04:58:18.329812+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[79733]: audit 2023-10-09T04:58:18.331868+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[79733]: cluster 2023-10-09T04:58:18.331950+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e334: 8 total, 7 up, 8 in 2023-10-09T04:58:19.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[79733]: cluster 2023-10-09T04:58:18.932007+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v1478: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:19.447 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[79733]: cluster 2023-10-09T04:58:19.047773+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:19.566 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:19 smithi153 bash[59565]: cluster 2023-10-09T04:58:18.329812+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:19 smithi153 bash[59565]: audit 2023-10-09T04:58:18.331868+0000 mon.a (mon.0) 1925 : audit 1 from='client.? 172.21.15.82:0/509075365' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:58:19.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:19 smithi153 bash[59565]: cluster 2023-10-09T04:58:18.331950+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e334: 8 total, 7 up, 8 in 2023-10-09T04:58:19.580 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:19 smithi153 bash[59565]: cluster 2023-10-09T04:58:18.932007+0000 mgr.y (mgr.34107) 1332 : cluster 0 pgmap v1478: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:19.581 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:19 smithi153 bash[59565]: cluster 2023-10-09T04:58:19.047773+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:19.602 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:19.705 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:19.727 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:19.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:19.830 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.861 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:19.955 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:19] "GET /metrics HTTP/1.1" 200 36512 "" "Prometheus/2.33.4" 2023-10-09T04:58:19.964 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:19.985 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:19.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:20.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:20.112 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:20.216 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.243 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:20.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:20.376 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:20.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.492 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:20 smithi153 bash[59565]: cluster 2023-10-09T04:58:19.047858+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:20.515 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:20.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:20.627 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:20 smithi082 bash[77063]: cluster 2023-10-09T04:58:19.047858+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:20.627 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:20 smithi082 bash[79733]: cluster 2023-10-09T04:58:19.047858+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T04:58:20.641 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:20.744 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.782 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:20.886 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.913 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:20.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:20.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:21.018 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:21.047 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:21.151 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.175 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:21.278 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:21.311 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:21.415 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:21.449 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:21.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:21 smithi082 bash[77063]: cluster 2023-10-09T04:58:20.932887+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v1479: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:21 smithi082 bash[79733]: cluster 2023-10-09T04:58:20.932887+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v1479: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:21.553 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.579 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:21 smithi153 bash[59565]: cluster 2023-10-09T04:58:20.932887+0000 mgr.y (mgr.34107) 1333 : cluster 0 pgmap v1479: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:21.682 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:21.710 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:21.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:21.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:21.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:21.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:22.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:22.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:22.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:22.195 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:22.222 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:22.302 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:58:22.302 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T04:58:22.302 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:58:22.303 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T04:58:22.303 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:58:22.303 DEBUG:teuthology.orchestra.run.smithi082:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:22.304 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T04:58:22.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:58:22.325 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:22.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi082.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:58:22.444 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:22.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:22.608 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:22.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:22.651 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:22.651 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:58:22.702 INFO:tasks.workunit:timeout=3h 2023-10-09T04:58:22.702 INFO:tasks.workunit:cleanup=True 2023-10-09T04:58:22.703 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:58:22.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:22.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:22.868 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:22.898 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:22.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:22.909 INFO:tasks.workunit.client.0.smithi082.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T04:58:23.002 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:23.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:23.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:23.125 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:23.159 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:23.262 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:23.297 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:23.401 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:23.433 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:23.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:23.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:23.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:23.665 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:23.696 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:23.800 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:23.830 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:23.934 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:23.959 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:23.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:24.062 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.087 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:24.190 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.198 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:23 smithi082 bash[77063]: cluster 2023-10-09T04:58:22.933583+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v1480: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:24.198 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:23 smithi082 bash[79733]: cluster 2023-10-09T04:58:22.933583+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v1480: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:24.217 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:24.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:24.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:23 smithi153 bash[59565]: cluster 2023-10-09T04:58:22.933583+0000 mgr.y (mgr.34107) 1334 : cluster 0 pgmap v1480: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:24.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:24.469 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:24.572 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:24.593 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:24.697 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:24.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:24.825 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.850 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:24.894 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:58:24.913 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:58:24.938 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:58:24.954 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:24.960 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:58:24.983 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:58:24.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:24.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:25.007 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:58:25.033 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:58:25.033 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T04:58:25.095 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.106 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:24 smithi082 bash[79733]: audit 2023-10-09T04:58:24.871479+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.82:0/4265066393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:25.106 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:24 smithi082 bash[77063]: audit 2023-10-09T04:58:24.871479+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.82:0/4265066393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:25.118 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:25.222 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:25.249 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:25.353 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:25.378 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:25.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:24 smithi153 bash[59565]: audit 2023-10-09T04:58:24.871479+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.82:0/4265066393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:25.481 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.510 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:25.613 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.638 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:25.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.762 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:25.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:25.866 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:25.897 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:25.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:26.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.025 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:26.128 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.154 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:26.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:26.269 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:26 smithi082 bash[77063]: cluster 2023-10-09T04:58:24.934038+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v1481: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:26.269 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:26 smithi082 bash[79733]: cluster 2023-10-09T04:58:24.934038+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v1481: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:26.285 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:26.388 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.410 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:26.513 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.537 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:26.640 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:26.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:26 smithi153 bash[59565]: cluster 2023-10-09T04:58:24.934038+0000 mgr.y (mgr.34107) 1335 : cluster 0 pgmap v1481: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:26.768 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:26.790 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:26.893 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.919 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:26.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:27.022 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:27.040 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:27.142 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:27.175 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:27.280 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.300 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:27.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:27.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.550 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:27.654 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:27.681 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:27.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:27.784 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:27.817 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:27.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:27.921 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:27.949 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:27.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:28.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:28.086 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:28.189 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:28.210 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is set 2023-10-09T04:58:28.214 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:28.317 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:28.329 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:28 smithi153 bash[59565]: cluster 2023-10-09T04:58:26.934533+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v1482: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:28.330 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:28 smithi153 bash[59565]: audit 2023-10-09T04:58:27.699700+0000 mon.a (mon.0) 1927 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:28.348 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:28.454 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:28.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:28 smithi082 bash[77063]: cluster 2023-10-09T04:58:26.934533+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v1482: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:28.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:28 smithi082 bash[77063]: audit 2023-10-09T04:58:27.699700+0000 mon.a (mon.0) 1927 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:28.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:28 smithi082 bash[79733]: cluster 2023-10-09T04:58:26.934533+0000 mgr.y (mgr.34107) 1336 : cluster 0 pgmap v1482: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:28.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:28 smithi082 bash[79733]: audit 2023-10-09T04:58:27.699700+0000 mon.a (mon.0) 1927 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:28.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:28.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:28.605 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:28.708 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:28.739 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:28.842 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:28.872 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:28.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:28.976 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.011 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:58:29.087 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:58:29.095 INFO:tasks.workunit.client.0.smithi082.stderr:Updating files: 96% (10741/11158) Updating files: 97% (10824/11158) Updating files: 98% (10935/11158) Updating files: 99% (11047/11158) Updating files: 100% (11158/11158) Updating files: 100% (11158/11158), done. 2023-10-09T04:58:29.112 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T04:58:29.114 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:29.146 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:29.159 INFO:tasks.workunit.client.0.smithi082.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T04:58:29.163 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:58:29.163 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T04:58:29.250 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:29.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:29 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.197267+0000 mon.a (mon.0) 1928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:29.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:29 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.202830+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.264 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:29 smithi153 bash[59565]: audit 2023-10-09T04:58:28.204387+0000 mon.a (mon.0) 1929 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:29.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:29 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.204581+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e335: 8 total, 7 up, 8 in 2023-10-09T04:58:29.265 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:29 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.348749+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.284 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:29.364 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T04:58:29.388 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.420 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.197267+0000 mon.a (mon.0) 1928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.202830+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[79733]: audit 2023-10-09T04:58:28.204387+0000 mon.a (mon.0) 1929 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.204581+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e335: 8 total, 7 up, 8 in 2023-10-09T04:58:29.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.348749+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.197267+0000 mon.a (mon.0) 1928 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:29.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.202830+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[77063]: audit 2023-10-09T04:58:28.204387+0000 mon.a (mon.0) 1929 : audit 1 from='client.? 172.21.15.82:0/2496384750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:29.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.204581+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e335: 8 total, 7 up, 8 in 2023-10-09T04:58:29.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.348749+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:29.493 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T04:58:29.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.557 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:29.609 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:58:29.612 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:58:29.612 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T04:58:29.661 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:29.686 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:29.770 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:58:29.777 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:58:29.777 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T04:58:29.789 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:29.822 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:29.862 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T04:58:29.864 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T04:58:29.864 DEBUG:teuthology.orchestra.run.smithi082:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T04:58:29.926 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:29.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:29.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:29.980 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:29] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T04:58:30.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:30.092 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:30.196 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:30.225 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:30.328 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.348913+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[77063]: cluster 2023-10-09T04:58:28.935028+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v1484: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[77063]: audit 2023-10-09T04:58:29.315707+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[77063]: audit 2023-10-09T04:58:29.316860+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.348913+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[79733]: cluster 2023-10-09T04:58:28.935028+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v1484: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:30.342 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[79733]: audit 2023-10-09T04:58:29.315707+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:30.343 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:30 smithi082 bash[79733]: audit 2023-10-09T04:58:29.316860+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:30.358 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:30.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:30.461 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:30.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:30 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.348913+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T04:58:30.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:30 smithi153 bash[59565]: cluster 2023-10-09T04:58:28.935028+0000 mgr.y (mgr.34107) 1337 : cluster 0 pgmap v1484: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:30.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:30 smithi153 bash[59565]: audit 2023-10-09T04:58:29.315707+0000 mon.a (mon.0) 1931 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:58:30.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:30 smithi153 bash[59565]: audit 2023-10-09T04:58:29.316860+0000 mon.a (mon.0) 1932 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:30.489 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:30.508 INFO:tasks.workunit.client.0.smithi082.stdout:Running main() from gmock_main.cc 2023-10-09T04:58:30.509 INFO:tasks.workunit.client.0.smithi082.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T04:58:30.509 INFO:tasks.workunit.client.0.smithi082.stdout:[----------] Global test environment set-up. 2023-10-09T04:58:30.509 INFO:tasks.workunit.client.0.smithi082.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T04:58:30.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:30.613 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:30.716 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:30.739 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:30.843 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:30.872 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:30.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:30.892 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:30.892 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:30.975 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:30.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:31.088 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:58:31.102 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.156 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:31.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:31.284 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:31.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:31.421 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:31.525 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:31.549 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:31.652 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:31.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:31.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:31.905 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.933 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:31.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:32.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:32.065 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:32.168 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:32.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:32.256 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T04:58:32.259 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_all_features (3 ms) 2023-10-09T04:58:32.259 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T04:58:32.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:32.344 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:32.448 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:32.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: cluster 2023-10-09T04:58:30.935916+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v1485: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:32.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: cluster 2023-10-09T04:58:31.232728+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: cluster 2023-10-09T04:58:31.234032+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e336: 8 total, 7 up, 8 in 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: cluster 2023-10-09T04:58:31.235113+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: audit 2023-10-09T04:58:31.332981+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.82:0/972240837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[77063]: audit 2023-10-09T04:58:31.333319+0000 mon.a (mon.0) 1934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: cluster 2023-10-09T04:58:30.935916+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v1485: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: cluster 2023-10-09T04:58:31.232728+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: cluster 2023-10-09T04:58:31.234032+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e336: 8 total, 7 up, 8 in 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: cluster 2023-10-09T04:58:31.235113+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.461 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: audit 2023-10-09T04:58:31.332981+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.82:0/972240837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.462 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:32 smithi082 bash[79733]: audit 2023-10-09T04:58:31.333319+0000 mon.a (mon.0) 1934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:32.492 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.copyup (233 ms) 2023-10-09T04:58:32.492 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T04:58:32.523 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.sparse_copyup (31 ms) 2023-10-09T04:58:32.523 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T04:58:32.541 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_and_set_id (18 ms) 2023-10-09T04:58:32.541 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T04:58:32.580 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:32.581 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.add_remove_child (39 ms) 2023-10-09T04:58:32.581 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T04:58:32.609 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: cluster 2023-10-09T04:58:30.935916+0000 mgr.y (mgr.34107) 1338 : cluster 0 pgmap v1485: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: cluster 2023-10-09T04:58:31.232728+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: cluster 2023-10-09T04:58:31.234032+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e336: 8 total, 7 up, 8 in 2023-10-09T04:58:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: cluster 2023-10-09T04:58:31.235113+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T04:58:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: audit 2023-10-09T04:58:31.332981+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.82:0/972240837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:32 smithi153 bash[59565]: audit 2023-10-09T04:58:31.333319+0000 mon.a (mon.0) 1934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:58:32.710 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.directory_methods (129 ms) 2023-10-09T04:58:32.710 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.create 2023-10-09T04:58:32.712 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:32.740 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:32.751 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.create (42 ms) 2023-10-09T04:58:32.751 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T04:58:32.754 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_features (3 ms) 2023-10-09T04:58:32.754 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T04:58:32.758 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_object_prefix (4 ms) 2023-10-09T04:58:32.758 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T04:58:32.760 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T04:58:32.761 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T04:58:32.763 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T04:58:32.763 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T04:58:32.765 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_modify_timestamp (3 ms) 2023-10-09T04:58:32.765 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T04:58:32.772 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_data_pool (7 ms) 2023-10-09T04:58:32.772 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T04:58:32.782 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.get_size (9 ms) 2023-10-09T04:58:32.782 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T04:58:32.791 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.set_size (10 ms) 2023-10-09T04:58:32.791 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T04:58:32.826 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.protection_status (35 ms) 2023-10-09T04:58:32.826 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T04:58:32.843 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.snapshot_limits (17 ms) 2023-10-09T04:58:32.843 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T04:58:32.845 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:32.878 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:32.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:32.982 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.017 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:33.118 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.parents_v1 (275 ms) 2023-10-09T04:58:33.118 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T04:58:33.120 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:33.152 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:33.175 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.parents_v2 (57 ms) 2023-10-09T04:58:33.175 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T04:58:33.219 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.snapshots (44 ms) 2023-10-09T04:58:33.219 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T04:58:33.229 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.snapid_race (9 ms) 2023-10-09T04:58:33.229 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T04:58:33.250 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.stripingv2 (22 ms) 2023-10-09T04:58:33.250 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T04:58:33.256 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:33.269 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_save (19 ms) 2023-10-09T04:58:33.269 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T04:58:33.278 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:33.281 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_resize (11 ms) 2023-10-09T04:58:33.281 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T04:58:33.291 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_update (10 ms) 2023-10-09T04:58:33.291 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T04:58:33.291 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-10-09T04:58:33.291 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T04:58:33.303 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_snap_add (12 ms) 2023-10-09T04:58:33.303 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T04:58:33.312 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-10-09T04:58:33.312 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T04:58:33.323 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.flags (12 ms) 2023-10-09T04:58:33.323 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T04:58:33.373 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.metadata (50 ms) 2023-10-09T04:58:33.374 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T04:58:33.382 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.396 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.set_features (23 ms) 2023-10-09T04:58:33.397 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T04:58:33.408 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:33.445 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:58:33.458 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror (61 ms) 2023-10-09T04:58:33.458 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T04:58:33.481 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror_image (24 ms) 2023-10-09T04:58:33.481 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T04:58:33.512 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.523 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.235903+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: audit 2023-10-09T04:58:32.237368+0000 mon.a (mon.0) 1935 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.237478+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e337: 8 total, 7 up, 8 in 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.240024+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.240384+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.242535+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.524 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:33 smithi153 bash[59565]: cluster 2023-10-09T04:58:33.242363+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e338: 8 total, 7 up, 8 in 2023-10-09T04:58:33.542 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:33.646 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:33.655 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.235903+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.655 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: audit 2023-10-09T04:58:32.237368+0000 mon.a (mon.0) 1935 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:58:33.655 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.237478+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e337: 8 total, 7 up, 8 in 2023-10-09T04:58:33.655 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.240024+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.240384+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.242535+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[77063]: cluster 2023-10-09T04:58:33.242363+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e338: 8 total, 7 up, 8 in 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.235903+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: audit 2023-10-09T04:58:32.237368+0000 mon.a (mon.0) 1935 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi082-243714-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.237478+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e337: 8 total, 7 up, 8 in 2023-10-09T04:58:33.656 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.240024+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.657 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.240384+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.657 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.242535+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T04:58:33.657 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:33 smithi082 bash[79733]: cluster 2023-10-09T04:58:33.242363+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e338: 8 total, 7 up, 8 in 2023-10-09T04:58:33.672 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:33.776 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.804 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:33.908 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.940 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:33.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:34.045 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:34.077 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:34.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:34.181 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:34.210 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:34.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:34.313 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:34.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:34 smithi153 bash[59565]: cluster 2023-10-09T04:58:32.936975+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v1488: 105 pgs: 23 unknown, 23 active+undersized, 12 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:58:34.324 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:34 smithi153 bash[59565]: cluster 2023-10-09T04:58:33.237563+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:34 smithi153 bash[59565]: cluster 2023-10-09T04:58:33.249028+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.325 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:34 smithi153 bash[59565]: cluster 2023-10-09T04:58:33.254304+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.341 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:34.446 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[77063]: cluster 2023-10-09T04:58:32.936975+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v1488: 105 pgs: 23 unknown, 23 active+undersized, 12 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:58:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[77063]: cluster 2023-10-09T04:58:33.237563+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[77063]: cluster 2023-10-09T04:58:33.249028+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[77063]: cluster 2023-10-09T04:58:33.254304+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[79733]: cluster 2023-10-09T04:58:32.936975+0000 mgr.y (mgr.34107) 1339 : cluster 0 pgmap v1488: 105 pgs: 23 unknown, 23 active+undersized, 12 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:58:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[79733]: cluster 2023-10-09T04:58:33.237563+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[79733]: cluster 2023-10-09T04:58:33.249028+0000 mon.a (mon.0) 1938 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:34.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:34 smithi082 bash[79733]: cluster 2023-10-09T04:58:33.254304+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T04:58:34.482 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:34.586 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:34.616 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:34.720 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:34.748 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:34.851 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:34.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:34.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:34.992 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:35.020 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:35.123 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:35.158 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:35.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:35.292 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:35.395 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:35.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:35.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:35.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:58:35.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:35.557 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:35.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:35.661 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:35.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:35.797 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:35.827 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:35.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:35.933 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:35.937 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:35.966 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:35.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:36.071 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.096 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:36.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.229 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:36 smithi082 bash[79733]: cluster 2023-10-09T04:58:34.937889+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v1490: 105 pgs: 10 unknown, 26 active+undersized, 16 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 696 KiB/s wr, 75 op/s; 23/135 objects degraded (17.037%) 2023-10-09T04:58:36.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:36 smithi082 bash[77063]: cluster 2023-10-09T04:58:34.937889+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v1490: 105 pgs: 10 unknown, 26 active+undersized, 16 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 696 KiB/s wr, 75 op/s; 23/135 objects degraded (17.037%) 2023-10-09T04:58:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:36.333 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.364 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:36.468 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.481 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:36 smithi153 bash[59565]: cluster 2023-10-09T04:58:34.937889+0000 mgr.y (mgr.34107) 1340 : cluster 0 pgmap v1490: 105 pgs: 10 unknown, 26 active+undersized, 16 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 696 KiB/s wr, 75 op/s; 23/135 objects degraded (17.037%) 2023-10-09T04:58:36.499 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:36.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:36.603 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.624 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:36.727 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:36.762 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:36.865 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:36.890 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:36.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:36.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:36.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.020 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:37.124 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.143 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:37.163 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror_image_status (3681 ms) 2023-10-09T04:58:37.163 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T04:58:37.246 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:37.280 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:37.295 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror_image_map (132 ms) 2023-10-09T04:58:37.295 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T04:58:37.305 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-10-09T04:58:37.305 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T04:58:37.330 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.mirror_snapshot (25 ms) 2023-10-09T04:58:37.330 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T04:58:37.333 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-10-09T04:58:37.334 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T04:58:37.337 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T04:58:37.337 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T04:58:37.341 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.dir_add_already_existing (4 ms) 2023-10-09T04:58:37.342 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T04:58:37.350 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_dir_rename (9 ms) 2023-10-09T04:58:37.350 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T04:58:37.355 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-09T04:58:37.355 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T04:58:37.361 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (6 ms) 2023-10-09T04:58:37.361 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T04:58:37.366 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-10-09T04:58:37.367 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T04:58:37.372 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_image_remove (7 ms) 2023-10-09T04:58:37.372 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T04:58:37.378 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_image_list (6 ms) 2023-10-09T04:58:37.378 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T04:58:37.383 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.386 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_image_clean (8 ms) 2023-10-09T04:58:37.386 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T04:58:37.390 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.image_group_add (4 ms) 2023-10-09T04:58:37.390 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T04:58:37.396 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.image_group_remove (6 ms) 2023-10-09T04:58:37.396 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T04:58:37.401 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.image_group_get (5 ms) 2023-10-09T04:58:37.401 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T04:58:37.404 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T04:58:37.404 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T04:58:37.407 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-09T04:58:37.407 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T04:58:37.411 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:37.412 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (5 ms) 2023-10-09T04:58:37.412 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T04:58:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:37.418 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (6 ms) 2023-10-09T04:58:37.418 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T04:58:37.422 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_set (4 ms) 2023-10-09T04:58:37.422 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T04:58:37.427 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_list (5 ms) 2023-10-09T04:58:37.428 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T04:58:37.461 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (34 ms) 2023-10-09T04:58:37.461 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T04:58:37.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:37.541 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:37.644 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:37.670 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:37.773 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:37.806 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:37.835 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (374 ms) 2023-10-09T04:58:37.836 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T04:58:37.842 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_remove (7 ms) 2023-10-09T04:58:37.843 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T04:58:37.846 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-10-09T04:58:37.847 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T04:58:37.865 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.trash_methods (19 ms) 2023-10-09T04:58:37.866 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T04:58:37.882 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.op_features (17 ms) 2023-10-09T04:58:37.882 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T04:58:37.909 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:37.922 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.clone_parent (39 ms) 2023-10-09T04:58:37.922 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T04:58:37.938 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:37.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:37.941 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.clone_parent_ns (19 ms) 2023-10-09T04:58:37.942 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T04:58:37.964 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.clone_child (23 ms) 2023-10-09T04:58:37.964 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T04:58:37.967 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:37.968 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:37.981 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.namespace_methods (16 ms) 2023-10-09T04:58:37.981 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T04:58:38.003 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.migration (23 ms) 2023-10-09T04:58:38.004 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T04:58:38.017 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-10-09T04:58:38.017 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T04:58:38.043 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:38.074 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:38.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:38.179 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:38.209 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:38.302 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:38 smithi082 bash[77063]: cluster 2023-10-09T04:58:36.938650+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v1491: 105 pgs: 28 active+undersized, 18 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 711 KiB/s wr, 142 op/s; 28/171 objects degraded (16.374%) 2023-10-09T04:58:38.302 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:38 smithi082 bash[79733]: cluster 2023-10-09T04:58:36.938650+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v1491: 105 pgs: 28 active+undersized, 18 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 711 KiB/s wr, 142 op/s; 28/171 objects degraded (16.374%) 2023-10-09T04:58:38.312 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:38.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:38.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:38 smithi153 bash[59565]: cluster 2023-10-09T04:58:36.938650+0000 mgr.y (mgr.34107) 1341 : cluster 0 pgmap v1491: 105 pgs: 28 active+undersized, 18 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.4 MiB/s rd, 711 KiB/s wr, 142 op/s; 28/171 objects degraded (16.374%) 2023-10-09T04:58:38.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:38.458 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:38.561 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:38.597 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:38.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:38.730 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:38.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:38.835 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:38.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:38.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:38.972 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:39.004 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:39.108 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.139 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.226 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1209 ms) 2023-10-09T04:58:39.226 INFO:tasks.workunit.client.0.smithi082.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T04:58:39.243 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.272 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.287 INFO:tasks.workunit.client.0.smithi082.stdout:[ OK ] TestClsRbd.sparsify (61 ms) 2023-10-09T04:58:39.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:39.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:39 smithi153 bash[59565]: cluster 2023-10-09T04:58:38.203691+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:39 smithi153 bash[59565]: cluster 2023-10-09T04:58:38.206171+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.389 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:39 smithi153 bash[59565]: cluster 2023-10-09T04:58:38.206328+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e339: 8 total, 7 up, 8 in 2023-10-09T04:58:39.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:39.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[77063]: cluster 2023-10-09T04:58:38.203691+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[77063]: cluster 2023-10-09T04:58:38.206171+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[77063]: cluster 2023-10-09T04:58:38.206328+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e339: 8 total, 7 up, 8 in 2023-10-09T04:58:39.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[79733]: cluster 2023-10-09T04:58:38.203691+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[79733]: cluster 2023-10-09T04:58:38.206171+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:39.513 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[79733]: cluster 2023-10-09T04:58:38.206328+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e339: 8 total, 7 up, 8 in 2023-10-09T04:58:39.544 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.648 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.683 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.788 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:39.816 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:39.919 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:39.950 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:39.951 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:39] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T04:58:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:39.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:40.053 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:40.082 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:40.185 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:40.214 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:40.236 INFO:tasks.workunit.client.0.smithi082.stdout:[----------] 70 tests from TestClsRbd (7031 ms total) 2023-10-09T04:58:40.236 INFO:tasks.workunit.client.0.smithi082.stdout: 2023-10-09T04:58:40.236 INFO:tasks.workunit.client.0.smithi082.stdout:[----------] Global test environment tear-down 2023-10-09T04:58:40.236 INFO:tasks.workunit.client.0.smithi082.stdout:[==========] 70 tests from 1 test suite ran. (9728 ms total) 2023-10-09T04:58:40.236 INFO:tasks.workunit.client.0.smithi082.stdout:[ PASSED ] 70 tests. 2023-10-09T04:58:40.239 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T04:58:40.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T04:58:40.301 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T04:58:40.301 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:58:40.318 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:40.369 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:40.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:38.448488+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:38.939715+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v1493: 105 pgs: 25 active+undersized, 21 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.13k op/s; 35/219 objects degraded (15.982%) 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:39.212163+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: Degraded data redundancy: 35/219 objects degraded (15.982%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:39.224362+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e340: 8 total, 7 up, 8 in 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:39.225852+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.484 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:40 smithi153 bash[59565]: cluster 2023-10-09T04:58:39.229474+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.501 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:40.537 DEBUG:teuthology.parallel:result is None 2023-10-09T04:58:40.537 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:38.448488+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:38.939715+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v1493: 105 pgs: 25 active+undersized, 21 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.13k op/s; 35/219 objects degraded (15.982%) 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:39.212163+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: Degraded data redundancy: 35/219 objects degraded (15.982%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:39.224362+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e340: 8 total, 7 up, 8 in 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:39.225852+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[79733]: cluster 2023-10-09T04:58:39.229474+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:38.448488+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T04:58:40.550 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:38.939715+0000 mgr.y (mgr.34107) 1342 : cluster 0 pgmap v1493: 105 pgs: 25 active+undersized, 21 active+undersized+degraded, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 2.13k op/s; 35/219 objects degraded (15.982%) 2023-10-09T04:58:40.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:39.212163+0000 mon.a (mon.0) 1940 : cluster 3 Health check update: Degraded data redundancy: 35/219 objects degraded (15.982%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T04:58:40.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:39.224362+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e340: 8 total, 7 up, 8 in 2023-10-09T04:58:40.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:39.225852+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.551 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:40 smithi082 bash[77063]: cluster 2023-10-09T04:58:39.229474+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:40.552 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:58:40.552 DEBUG:teuthology.orchestra.run.smithi082:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:40.600 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:58:40.600 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:58:40.601 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T04:58:40.601 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:58:40.601 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T04:58:40.601 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:58:40.601 DEBUG:teuthology.orchestra.run.smithi082:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:40.606 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.648 INFO:teuthology.orchestra.run.smithi082.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:58:40.649 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:40.665 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:40.670 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:58:40.670 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:58:40.719 INFO:tasks.workunit:timeout=3h 2023-10-09T04:58:40.719 INFO:tasks.workunit:cleanup=True 2023-10-09T04:58:40.720 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:58:40.768 INFO:tasks.workunit.client.0.smithi082.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T04:58:40.769 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:40.794 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:40.898 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:40.932 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:41.036 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:41.069 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:41.172 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:41.204 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:41.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:41.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[79733]: cluster 2023-10-09T04:58:39.803008+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:41.316 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[79733]: cluster 2023-10-09T04:58:40.229742+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:41.317 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[79733]: cluster 2023-10-09T04:58:40.233300+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e341: 8 total, 7 up, 8 in 2023-10-09T04:58:41.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[77063]: cluster 2023-10-09T04:58:39.803008+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:41.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[77063]: cluster 2023-10-09T04:58:40.229742+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:41.317 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:41 smithi082 bash[77063]: cluster 2023-10-09T04:58:40.233300+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e341: 8 total, 7 up, 8 in 2023-10-09T04:58:41.333 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:41.436 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:41.449 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:41 smithi153 bash[59565]: cluster 2023-10-09T04:58:39.803008+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T04:58:41.449 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:41 smithi153 bash[59565]: cluster 2023-10-09T04:58:40.229742+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:41.449 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:41 smithi153 bash[59565]: cluster 2023-10-09T04:58:40.233300+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e341: 8 total, 7 up, 8 in 2023-10-09T04:58:41.469 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:41.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:41.604 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:41.698 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:41.707 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:41.732 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:41.835 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:41.868 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:41.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:41.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:41.998 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:42.101 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:42.128 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:42.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:42.232 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:42.256 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:42.360 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:42.380 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:42.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:42.484 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:42.497 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:42 smithi153 bash[59565]: cluster 2023-10-09T04:58:40.650129+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.497 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:42 smithi153 bash[59565]: cluster 2023-10-09T04:58:40.652985+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.497 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:42 smithi153 bash[59565]: cluster 2023-10-09T04:58:40.940608+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v1496: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:42.497 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:42 smithi153 bash[59565]: cluster 2023-10-09T04:58:41.151965+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[79733]: cluster 2023-10-09T04:58:40.650129+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[79733]: cluster 2023-10-09T04:58:40.652985+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[79733]: cluster 2023-10-09T04:58:40.940608+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v1496: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[79733]: cluster 2023-10-09T04:58:41.151965+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[77063]: cluster 2023-10-09T04:58:40.650129+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[77063]: cluster 2023-10-09T04:58:40.652985+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.512 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[77063]: cluster 2023-10-09T04:58:40.940608+0000 mgr.y (mgr.34107) 1343 : cluster 0 pgmap v1496: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:42.513 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:42 smithi082 bash[77063]: cluster 2023-10-09T04:58:41.151965+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T04:58:42.514 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:42.618 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:42.642 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:42.745 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:42.778 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:42.882 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:42.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:42.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:43.006 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.040 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:43.144 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:43.166 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:43.270 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.305 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:43.409 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.442 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:43.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:43.581 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:43.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:43.685 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:43.709 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:43.812 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:43.834 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:43.937 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:43.965 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:43.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:44.069 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.101 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:44.204 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.239 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:44.344 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:44.351 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:44 smithi082 bash[77063]: cluster 2023-10-09T04:58:42.941673+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v1497: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:44.351 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:44 smithi082 bash[79733]: cluster 2023-10-09T04:58:42.941673+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v1497: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:44.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:44.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:44.488 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:44.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.604 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:44 smithi153 bash[59565]: cluster 2023-10-09T04:58:42.941673+0000 mgr.y (mgr.34107) 1344 : cluster 0 pgmap v1497: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:44.622 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:44.726 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.757 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:44.861 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:44.891 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:44.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:44.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.018 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:45.121 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:45.154 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:45.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.282 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:45.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.409 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:45 smithi153 bash[59565]: audit 2023-10-09T04:58:44.308230+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:45 smithi153 bash[59565]: cluster 2023-10-09T04:58:45.197061+0000 mon.a (mon.0) 1944 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:45.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.523 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:45 smithi082 bash[77063]: audit 2023-10-09T04:58:44.308230+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:45.523 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:45 smithi082 bash[77063]: cluster 2023-10-09T04:58:45.197061+0000 mon.a (mon.0) 1944 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:45.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:45 smithi082 bash[79733]: audit 2023-10-09T04:58:44.308230+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:45.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:45 smithi082 bash[79733]: cluster 2023-10-09T04:58:45.197061+0000 mon.a (mon.0) 1944 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:45.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:45.639 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:45.672 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:45.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:45.814 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:45.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:45.950 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:45.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:46.055 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:46.080 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:46.152 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T04:58:46.160 INFO:tasks.workunit.client.0.smithi082.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T04:58:46.162 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:58:46.162 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T04:58:46.183 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:46.207 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:46.310 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:46.321 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:46 smithi082 bash[77063]: cluster 2023-10-09T04:58:44.942882+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v1498: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:46.322 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:46 smithi082 bash[79733]: cluster 2023-10-09T04:58:44.942882+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v1498: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:46.337 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T04:58:46.338 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:46.442 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:46.461 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:46.564 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:46.587 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:46 smithi153 bash[59565]: cluster 2023-10-09T04:58:44.942882+0000 mgr.y (mgr.34107) 1345 : cluster 0 pgmap v1498: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:46.690 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:46.722 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:46.826 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:46.848 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:46.951 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:46.986 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:46.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:47.020 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T04:58:47.089 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:47.123 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:47.227 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:47.264 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:47.265 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:58:47.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:47.267 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:58:47.267 INFO:tasks.workunit.client.0.smithi082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T04:58:47.367 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:47.392 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:47.459 INFO:tasks.workunit.client.0.smithi082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:58:47.463 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2023-10-09T04:58:47.463 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T04:58:47.468 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T04:58:47.469 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T04:58:47.469 DEBUG:teuthology.orchestra.run.smithi082:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T04:58:47.496 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:47.516 INFO:tasks.workunit.client.0.smithi082.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T04:58:47.517 INFO:tasks.workunit.client.0.smithi082.stderr:+ TMPDIR=/tmp/rbd_import_export_244924 2023-10-09T04:58:47.517 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm -rf /tmp/rbd_import_export_244924 2023-10-09T04:58:47.517 INFO:tasks.workunit.client.0.smithi082.stderr:+ mkdir /tmp/rbd_import_export_244924 2023-10-09T04:58:47.518 INFO:tasks.workunit.client.0.smithi082.stderr:+ trap rm -rf /tmp/rbd_import_export_244924 INT TERM EXIT 2023-10-09T04:58:47.518 INFO:tasks.workunit.client.0.smithi082.stderr:+ mkdir foo.244924 2023-10-09T04:58:47.519 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import foo.244924 foo.dir 2023-10-09T04:58:47.535 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:47.639 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:47.661 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:47.720 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: cannot import a directory 2023-10-09T04:58:47.720 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 0% complete...failed. 2023-10-09T04:58:47.720 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: import failed: (21) Is a directory 2023-10-09T04:58:47.764 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:47.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:47.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:47.936 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:47.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:48.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:48.067 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:48.084 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:58:48.084 INFO:tasks.workunit.client.0.smithi082.stderr:+ rmdir foo.244924 2023-10-09T04:58:48.086 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_244924/img bs=1k count=1 seek=10 2023-10-09T04:58:48.087 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:58:48.087 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:58:48.087 INFO:tasks.workunit.client.0.smithi082.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000131304 s, 7.8 MB/s 2023-10-09T04:58:48.087 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_244924/img bs=1k count=10 seek=100 2023-10-09T04:58:48.088 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records in 2023-10-09T04:58:48.088 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records out 2023-10-09T04:58:48.088 INFO:tasks.workunit.client.0.smithi082.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000105627 s, 96.9 MB/s 2023-10-09T04:58:48.088 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_244924/img bs=1k count=100 seek=1000 2023-10-09T04:58:48.090 INFO:tasks.workunit.client.0.smithi082.stderr:70+1 records in 2023-10-09T04:58:48.090 INFO:tasks.workunit.client.0.smithi082.stderr:70+1 records out 2023-10-09T04:58:48.090 INFO:tasks.workunit.client.0.smithi082.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000694034 s, 104 MB/s 2023-10-09T04:58:48.090 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_244924/img bs=1k seek=10000 2023-10-09T04:58:48.091 INFO:tasks.workunit.client.0.smithi082.stderr:138+1 records in 2023-10-09T04:58:48.091 INFO:tasks.workunit.client.0.smithi082.stderr:138+1 records out 2023-10-09T04:58:48.092 INFO:tasks.workunit.client.0.smithi082.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000652863 s, 218 MB/s 2023-10-09T04:58:48.092 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_244924/img bs=1k seek=100000 2023-10-09T04:58:48.170 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:48.186 INFO:tasks.workunit.client.0.smithi082.stderr:74+1 records in 2023-10-09T04:58:48.186 INFO:tasks.workunit.client.0.smithi082.stderr:74+1 records out 2023-10-09T04:58:48.186 INFO:tasks.workunit.client.0.smithi082.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.0944097 s, 807 kB/s 2023-10-09T04:58:48.187 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_244924/img bs=1k seek=1000000 2023-10-09T04:58:48.200 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:48.250 INFO:tasks.workunit.client.0.smithi082.stderr:194+1 records in 2023-10-09T04:58:48.250 INFO:tasks.workunit.client.0.smithi082.stderr:194+1 records out 2023-10-09T04:58:48.250 INFO:tasks.workunit.client.0.smithi082.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.062019 s, 3.2 MB/s 2023-10-09T04:58:48.250 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:58:48.296 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:58:48.296 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:58:48.299 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:58:48.299 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:58:48.304 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:48.334 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:48.439 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:48.446 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:48 smithi082 bash[77063]: cluster 2023-10-09T04:58:46.944278+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v1499: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:48.446 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:48 smithi082 bash[79733]: cluster 2023-10-09T04:58:46.944278+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v1499: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:48.591 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:48 smithi153 bash[59565]: cluster 2023-10-09T04:58:46.944278+0000 mgr.y (mgr.34107) 1346 : cluster 0 pgmap v1499: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:48.694 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:48.723 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:48.827 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:48.851 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:48.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:48.954 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:48.978 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:48.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:48.986 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T04:58:48.987 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:58:48.990 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg /tmp/rbd_import_export_244924/img2 2023-10-09T04:58:49.083 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:49.104 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T04:58:49.106 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T04:58:49.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:49.219 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:49.249 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:49.324 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:58:49.352 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:49.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:49.400 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg - 2023-10-09T04:58:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:49 smithi153 bash[59565]: cluster 2023-10-09T04:58:48.944999+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v1500: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:49 smithi153 bash[59565]: audit 2023-10-09T04:58:49.092185+0000 mgr.y (mgr.34107) 1348 : audit 0 from='client.36102 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:49.472 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:49.500 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:49.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:49.638 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:49.698 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:49 smithi082 bash[79733]: cluster 2023-10-09T04:58:48.944999+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v1500: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:49.698 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:49 smithi082 bash[79733]: audit 2023-10-09T04:58:49.092185+0000 mgr.y (mgr.34107) 1348 : audit 0 from='client.36102 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:49.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:49 smithi082 bash[77063]: cluster 2023-10-09T04:58:48.944999+0000 mgr.y (mgr.34107) 1347 : cluster 0 pgmap v1500: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 17/84 objects degraded (20.238%) 2023-10-09T04:58:49.698 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:49 smithi082 bash[77063]: audit 2023-10-09T04:58:49.092185+0000 mgr.y (mgr.34107) 1348 : audit 0 from='client.36102 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:49.743 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:49.767 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:49.871 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:49.894 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:49.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:50.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:50.003 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:49] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T04:58:50.024 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:50.127 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:50.152 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:50.256 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:50.288 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:50.334 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:58:50.338 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:58:50.392 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.423 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:50.473 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:58:50.477 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img2 2023-10-09T04:58:50.527 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.558 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:50.662 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.693 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:50.798 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:50.830 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:50.933 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:50.962 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:50.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:51.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:51.091 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:51.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:51.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:51.248 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img3 2023-10-09T04:58:51.295 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:51.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:51.398 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:51.434 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:51.538 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:51.549 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:51 smithi082 bash[77063]: cluster 2023-10-09T04:58:51.201537+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:51.549 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:51 smithi082 bash[79733]: cluster 2023-10-09T04:58:51.201537+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:51.565 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:51.669 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:51 smithi153 bash[59565]: cluster 2023-10-09T04:58:51.201537+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 10 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:51.696 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:51.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:51.818 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img2 /tmp/rbd_import_export_244924/img3 2023-10-09T04:58:51.830 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:51.932 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:51.961 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:51.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:52.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:52.094 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:52.197 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:52.234 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:52.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:52.339 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:52.370 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:52.474 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:52.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:52 smithi082 bash[77063]: cluster 2023-10-09T04:58:50.945640+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v1501: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.8 KiB/s rd, 95 B/s wr, 7 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:52.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:52 smithi082 bash[79733]: cluster 2023-10-09T04:58:50.945640+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v1501: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.8 KiB/s rd, 95 B/s wr, 7 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:52.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:52.605 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:52.637 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:52.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:52 smithi153 bash[59565]: cluster 2023-10-09T04:58:50.945640+0000 mgr.y (mgr.34107) 1349 : cluster 0 pgmap v1501: 73 pgs: 20 active+undersized, 10 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.8 KiB/s rd, 95 B/s wr, 7 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:52.741 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:52.771 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:52.876 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:52.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:53.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:53.033 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:53.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:53.164 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:53.268 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:53.298 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:53.401 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:53.435 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:53.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:53 smithi082 bash[79733]: cluster 2023-10-09T04:58:52.946865+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v1502: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:53.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:53 smithi082 bash[77063]: cluster 2023-10-09T04:58:52.946865+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v1502: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:53.540 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:53.570 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:53.673 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:53.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:53 smithi153 bash[59565]: cluster 2023-10-09T04:58:52.946865+0000 mgr.y (mgr.34107) 1350 : cluster 0 pgmap v1502: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:53.706 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:53.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:53.810 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:53.844 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:53.948 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:53.983 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:53.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:54.087 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:54.119 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:54.222 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.257 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:54.361 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:54.386 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:54.489 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.523 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:54.626 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:54.661 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:54.765 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:54.797 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:54.901 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.929 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:54.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:55.032 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.059 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:55.162 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:55.195 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:55.299 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:55.332 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:55.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:55.436 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:55.462 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:55.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:55.565 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:55.599 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:55.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:55.736 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:55.839 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.874 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:55.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:55.978 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:56.006 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:56.110 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:56.137 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:56.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:56.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:56.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:56.298 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format - testimg 2023-10-09T04:58:56.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.406 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:56 smithi082 bash[77063]: cluster 2023-10-09T04:58:54.948041+0000 mgr.y (mgr.34107) 1351 : cluster 0 pgmap v1503: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:56.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:56 smithi082 bash[79733]: cluster 2023-10-09T04:58:54.948041+0000 mgr.y (mgr.34107) 1351 : cluster 0 pgmap v1503: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:56.511 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:56.536 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:56.640 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.653 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:56 smithi153 bash[59565]: cluster 2023-10-09T04:58:54.948041+0000 mgr.y (mgr.34107) 1351 : cluster 0 pgmap v1503: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 KiB/s rd, 43 KiB/s wr, 16 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:58:56.674 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:56.777 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:56.802 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:56.873 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:58:56.874 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:58:56.877 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg /tmp/rbd_import_export_244924/img2 2023-10-09T04:58:56.907 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.934 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:56.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:57.037 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:57.066 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:57.170 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:57.197 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:57.260 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:58:57.264 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg - 2023-10-09T04:58:57.300 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:57.329 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:57.433 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:57.466 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:57 smithi082 bash[77063]: cluster 2023-10-09T04:58:56.948997+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v1504: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 101 KiB/s rd, 43 KiB/s wr, 25 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:57.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:58:57 smithi082 bash[77063]: cluster 2023-10-09T04:58:57.205100+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:57.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:57 smithi082 bash[79733]: cluster 2023-10-09T04:58:56.948997+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v1504: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 101 KiB/s rd, 43 KiB/s wr, 25 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:57.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:58:57 smithi082 bash[79733]: cluster 2023-10-09T04:58:57.205100+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:57.570 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:57.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:57 smithi153 bash[59565]: cluster 2023-10-09T04:58:56.948997+0000 mgr.y (mgr.34107) 1352 : cluster 0 pgmap v1504: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 101 KiB/s rd, 43 KiB/s wr, 25 op/s; 17/84 objects degraded (20.238%) 2023-10-09T04:58:57.583 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:58:57 smithi153 bash[59565]: cluster 2023-10-09T04:58:57.205100+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: Degraded data redundancy: 17/84 objects degraded (20.238%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:57.601 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:57.704 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:57.734 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:57.837 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:57.867 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:57.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:57.971 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.002 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:58.106 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.140 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:58.179 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:58:58.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:58:58.244 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:58.271 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:58.318 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:58:58.322 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img2 2023-10-09T04:58:58.375 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:58.407 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:58.511 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.545 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:58.649 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:58.674 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:58.778 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.806 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:58.909 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:58.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:58.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:59.013 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img3 2023-10-09T04:58:59.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:59.073 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:59.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:59.175 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.205 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:59.309 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.312 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1500,"stamp":"2023-10-09T04:58:48.944524+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6287,"num_read_kb":12723,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":8398258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1171,"ondisk_log_size":1171,"up":189,"acting":189,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6739196,"kb_used_data":294348,"kb_used_omap":47,"kb_used_meta":6444624,"kb_avail":743222020,"statfs":{"total":767960285184,"available":761059348480,"internally_reserved":0,"allocated":301412352,"data_stored":293954273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48739,"internal_metadata":6599295389},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374,"num_objects":-47,"num_object_clones":0,"num_object_copies":-141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-47,"num_whiteouts":0,"num_read":-9749,"num_read_kb":-18788,"num_write":-4590,"num_write_kb":-8624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-38,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.723966"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.808846+0000","last_change":"2023-10-09T04:56:46.364962+0000","last_active":"2023-10-09T04:58:39.808846+0000","last_peered":"2023-10-09T04:58:39.808846+0000","last_clean":"2023-10-09T04:58:39.808846+0000","last_became_active":"2023-10-09T04:56:46.363050+0000","last_became_peered":"2023-10-09T04:56:46.363050+0000","last_unstale":"2023-10-09T04:58:39.808846+0000","last_undegraded":"2023-10-09T04:58:39.808846+0000","last_fullsized":"2023-10-09T04:58:39.808846+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":314,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810378+0000","last_change":"2023-10-09T04:57:17.365284+0000","last_active":"2023-10-09T04:58:39.810378+0000","last_peered":"2023-10-09T04:58:39.810378+0000","last_clean":"2023-10-09T04:58:39.810378+0000","last_became_active":"2023-10-09T04:57:17.364979+0000","last_became_peered":"2023-10-09T04:57:17.364979+0000","last_unstale":"2023-10-09T04:58:39.810378+0000","last_undegraded":"2023-10-09T04:58:39.810378+0000","last_fullsized":"2023-10-09T04:58:39.810378+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":316,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.937999+0000","last_change":"2023-10-09T04:56:46.354560+0000","last_active":"2023-10-09T04:58:39.937999+0000","last_peered":"2023-10-09T04:58:39.937999+0000","last_clean":"2023-10-09T04:58:39.937999+0000","last_became_active":"2023-10-09T04:56:46.353658+0000","last_became_peered":"2023-10-09T04:56:46.353658+0000","last_unstale":"2023-10-09T04:58:39.937999+0000","last_undegraded":"2023-10-09T04:58:39.937999+0000","last_fullsized":"2023-10-09T04:58:39.937999+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:19.859253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":329,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.404734+0000","last_change":"2023-10-09T04:57:42.203481+0000","last_active":"2023-10-09T04:58:39.404734+0000","last_peered":"2023-10-09T04:58:39.404734+0000","last_clean":"2023-10-09T04:57:27.246048+0000","last_became_active":"2023-10-09T04:57:42.203481+0000","last_became_peered":"2023-10-09T04:57:42.203481+0000","last_unstale":"2023-10-09T04:58:39.404734+0000","last_undegraded":"2023-10-09T04:58:39.404734+0000","last_fullsized":"2023-10-09T04:57:42.194703+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:44.603444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":325,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.403649+0000","last_change":"2023-10-09T04:28:51.529288+0000","last_active":"2023-10-09T04:58:39.403649+0000","last_peered":"2023-10-09T04:58:39.403649+0000","last_clean":"2023-10-09T04:58:39.403649+0000","last_became_active":"2023-10-09T04:28:51.528842+0000","last_became_peered":"2023-10-09T04:28:51.528842+0000","last_unstale":"2023-10-09T04:58:39.403649+0000","last_undegraded":"2023-10-09T04:58:39.403649+0000","last_fullsized":"2023-10-09T04:58:39.403649+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:35.322840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":296,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.237952+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T04:58:39.237952+0000","last_peered":"2023-10-09T04:58:39.237952+0000","last_clean":"2023-10-09T04:58:39.237952+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T04:58:39.237952+0000","last_undegraded":"2023-10-09T04:58:39.237952+0000","last_fullsized":"2023-10-09T04:58:39.237952+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":317,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T04:58:40.236470+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T04:58:40.236470+0000","last_peered":"2023-10-09T04:58:40.236470+0000","last_clean":"2023-10-09T04:58:40.236470+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T04:58:40.236470+0000","last_undegraded":"2023-10-09T04:58:40.236470+0000","last_fullsized":"2023-10-09T04:58:40.236470+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.937987+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T04:58:39.937987+0000","last_peered":"2023-10-09T04:58:39.937987+0000","last_clean":"2023-10-09T04:58:39.937987+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T04:58:39.937987+0000","last_undegraded":"2023-10-09T04:58:39.937987+0000","last_fullsized":"2023-10-09T04:58:39.937987+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":288,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.234189+0000","last_change":"2023-10-09T04:57:42.207503+0000","last_active":"2023-10-09T04:58:39.234189+0000","last_peered":"2023-10-09T04:58:39.234189+0000","last_clean":"2023-10-09T04:57:27.234558+0000","last_became_active":"2023-10-09T04:57:42.207503+0000","last_became_peered":"2023-10-09T04:57:42.207503+0000","last_unstale":"2023-10-09T04:58:39.234189+0000","last_undegraded":"2023-10-09T04:58:39.234189+0000","last_fullsized":"2023-10-09T04:57:42.201741+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":143,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.236706+0000","last_change":"2023-10-09T04:57:42.205998+0000","last_active":"2023-10-09T04:58:39.236706+0000","last_peered":"2023-10-09T04:58:39.236706+0000","last_clean":"2023-10-09T04:33:01.723744+0000","last_became_active":"2023-10-09T04:57:42.205998+0000","last_became_peered":"2023-10-09T04:57:42.205998+0000","last_unstale":"2023-10-09T04:58:39.236706+0000","last_undegraded":"2023-10-09T04:58:39.236706+0000","last_fullsized":"2023-10-09T04:57:42.202540+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":39,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.938754+0000","last_change":"2023-10-09T04:57:42.212481+0000","last_active":"2023-10-09T04:58:39.938754+0000","last_peered":"2023-10-09T04:58:39.938754+0000","last_clean":"2023-10-09T04:11:28.924478+0000","last_became_active":"2023-10-09T04:57:42.212481+0000","last_became_peered":"2023-10-09T04:57:42.212481+0000","last_unstale":"2023-10-09T04:58:39.938754+0000","last_undegraded":"2023-10-09T04:57:42.198065+0000","last_fullsized":"2023-10-09T04:57:42.197952+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:14.689295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":284,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.236853+0000","last_change":"2023-10-09T04:50:44.052710+0000","last_active":"2023-10-09T04:58:39.236853+0000","last_peered":"2023-10-09T04:58:39.236853+0000","last_clean":"2023-10-09T04:58:39.236853+0000","last_became_active":"2023-10-09T04:50:44.052312+0000","last_became_peered":"2023-10-09T04:50:44.052312+0000","last_unstale":"2023-10-09T04:58:39.236853+0000","last_undegraded":"2023-10-09T04:58:39.236853+0000","last_fullsized":"2023-10-09T04:58:39.236853+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":54,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.809367+0000","last_change":"2023-10-09T04:57:42.211357+0000","last_active":"2023-10-09T04:58:39.809367+0000","last_peered":"2023-10-09T04:58:39.809367+0000","last_clean":"2023-10-09T04:57:27.235502+0000","last_became_active":"2023-10-09T04:57:42.211357+0000","last_became_peered":"2023-10-09T04:57:42.211357+0000","last_unstale":"2023-10-09T04:58:39.809367+0000","last_undegraded":"2023-10-09T04:57:42.209081+0000","last_fullsized":"2023-10-09T04:57:42.208941+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810388+0000","last_change":"2023-10-09T04:56:46.360074+0000","last_active":"2023-10-09T04:58:39.810388+0000","last_peered":"2023-10-09T04:58:39.810388+0000","last_clean":"2023-10-09T04:58:39.810388+0000","last_became_active":"2023-10-09T04:56:46.359135+0000","last_became_peered":"2023-10-09T04:56:46.359135+0000","last_unstale":"2023-10-09T04:58:39.810388+0000","last_undegraded":"2023-10-09T04:58:39.810388+0000","last_fullsized":"2023-10-09T04:58:39.810388+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":192,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.809412+0000","last_change":"2023-10-09T04:57:42.221613+0000","last_active":"2023-10-09T04:58:39.809412+0000","last_peered":"2023-10-09T04:58:39.809412+0000","last_clean":"2023-10-09T04:56:42.208491+0000","last_became_active":"2023-10-09T04:57:42.221613+0000","last_became_peered":"2023-10-09T04:57:42.221613+0000","last_unstale":"2023-10-09T04:58:39.809412+0000","last_undegraded":"2023-10-09T04:57:42.209950+0000","last_fullsized":"2023-10-09T04:57:42.209819+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.938570+0000","last_change":"2023-10-09T04:57:42.201885+0000","last_active":"2023-10-09T04:58:39.938570+0000","last_peered":"2023-10-09T04:58:39.938570+0000","last_clean":"2023-10-09T04:40:38.976160+0000","last_became_active":"2023-10-09T04:57:42.201885+0000","last_became_peered":"2023-10-09T04:57:42.201885+0000","last_unstale":"2023-10-09T04:58:39.938570+0000","last_undegraded":"2023-10-09T04:58:39.938570+0000","last_fullsized":"2023-10-09T04:57:42.197909+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:32.314819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":326,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.404039+0000","last_change":"2023-10-09T04:57:42.202997+0000","last_active":"2023-10-09T04:58:39.404039+0000","last_peered":"2023-10-09T04:58:39.404039+0000","last_clean":"2023-10-09T04:57:27.650559+0000","last_became_active":"2023-10-09T04:57:42.202997+0000","last_became_peered":"2023-10-09T04:57:42.202997+0000","last_unstale":"2023-10-09T04:58:39.404039+0000","last_undegraded":"2023-10-09T04:58:39.404039+0000","last_fullsized":"2023-10-09T04:57:42.195253+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:03.887139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":306,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.234280+0000","last_change":"2023-10-09T04:57:42.207717+0000","last_active":"2023-10-09T04:58:39.234280+0000","last_peered":"2023-10-09T04:58:39.234280+0000","last_clean":"2023-10-09T04:57:27.230333+0000","last_became_active":"2023-10-09T04:57:42.207717+0000","last_became_peered":"2023-10-09T04:57:42.207717+0000","last_unstale":"2023-10-09T04:58:39.234280+0000","last_undegraded":"2023-10-09T04:58:39.234280+0000","last_fullsized":"2023-10-09T04:57:42.201259+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.808294+0000","last_change":"2023-10-09T04:57:42.214569+0000","last_active":"2023-10-09T04:58:39.808294+0000","last_peered":"2023-10-09T04:58:39.808294+0000","last_clean":"2023-10-09T04:56:42.209033+0000","last_became_active":"2023-10-09T04:57:42.214569+0000","last_became_peered":"2023-10-09T04:57:42.214569+0000","last_unstale":"2023-10-09T04:58:39.808294+0000","last_undegraded":"2023-10-09T04:58:39.808294+0000","last_fullsized":"2023-10-09T04:57:42.210713+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":326,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.404742+0000","last_change":"2023-10-09T04:57:42.203083+0000","last_active":"2023-10-09T04:58:39.404742+0000","last_peered":"2023-10-09T04:58:39.404742+0000","last_clean":"2023-10-09T04:57:27.246132+0000","last_became_active":"2023-10-09T04:57:42.203083+0000","last_became_peered":"2023-10-09T04:57:42.203083+0000","last_unstale":"2023-10-09T04:58:39.404742+0000","last_undegraded":"2023-10-09T04:58:39.404742+0000","last_fullsized":"2023-10-09T04:57:42.195174+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:50.100582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":305,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.236445+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T04:58:39.236445+0000","last_peered":"2023-10-09T04:58:39.236445+0000","last_clean":"2023-10-09T04:58:39.236445+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T04:58:39.236445+0000","last_undegraded":"2023-10-09T04:58:39.236445+0000","last_fullsized":"2023-10-09T04:58:39.236445+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":243,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.939411+0000","last_change":"2023-10-09T04:57:42.204104+0000","last_active":"2023-10-09T04:58:39.939411+0000","last_peered":"2023-10-09T04:58:39.939411+0000","last_clean":"2023-10-09T04:57:27.239550+0000","last_became_active":"2023-10-09T04:57:42.204104+0000","last_became_peered":"2023-10-09T04:57:42.204104+0000","last_unstale":"2023-10-09T04:58:39.939411+0000","last_undegraded":"2023-10-09T04:58:39.939411+0000","last_fullsized":"2023-10-09T04:57:42.199809+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:06.735981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":173,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.810115+0000","last_change":"2023-10-09T04:57:42.218881+0000","last_active":"2023-10-09T04:58:39.810115+0000","last_peered":"2023-10-09T04:58:39.810115+0000","last_clean":"2023-10-09T04:56:42.211526+0000","last_became_active":"2023-10-09T04:57:42.218881+0000","last_became_peered":"2023-10-09T04:57:42.218881+0000","last_unstale":"2023-10-09T04:58:39.810115+0000","last_undegraded":"2023-10-09T04:57:42.209658+0000","last_fullsized":"2023-10-09T04:57:42.209532+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.233585+0000","last_change":"2023-10-09T04:57:42.205170+0000","last_active":"2023-10-09T04:58:39.233585+0000","last_peered":"2023-10-09T04:58:39.233585+0000","last_clean":"2023-10-09T04:57:27.230491+0000","last_became_active":"2023-10-09T04:57:42.205170+0000","last_became_peered":"2023-10-09T04:57:42.205170+0000","last_unstale":"2023-10-09T04:58:39.233585+0000","last_undegraded":"2023-10-09T04:58:39.233585+0000","last_fullsized":"2023-10-09T04:57:42.200345+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":186,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.809668+0000","last_change":"2023-10-09T04:57:42.217453+0000","last_active":"2023-10-09T04:58:39.809668+0000","last_peered":"2023-10-09T04:58:39.809668+0000","last_clean":"2023-10-09T04:56:42.211808+0000","last_became_active":"2023-10-09T04:57:42.217453+0000","last_became_peered":"2023-10-09T04:57:42.217453+0000","last_unstale":"2023-10-09T04:58:39.809668+0000","last_undegraded":"2023-10-09T04:57:42.212085+0000","last_fullsized":"2023-10-09T04:57:42.211906+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.808984+0000","last_change":"2023-10-09T04:56:46.351680+0000","last_active":"2023-10-09T04:58:39.808984+0000","last_peered":"2023-10-09T04:58:39.808984+0000","last_clean":"2023-10-09T04:58:39.808984+0000","last_became_active":"2023-10-09T04:56:46.351117+0000","last_became_peered":"2023-10-09T04:56:46.351117+0000","last_unstale":"2023-10-09T04:58:39.808984+0000","last_undegraded":"2023-10-09T04:58:39.808984+0000","last_fullsized":"2023-10-09T04:58:39.808984+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.234730+0000","last_change":"2023-10-09T04:57:42.205037+0000","last_active":"2023-10-09T04:58:39.234730+0000","last_peered":"2023-10-09T04:58:39.234730+0000","last_clean":"2023-10-09T04:40:15.039200+0000","last_became_active":"2023-10-09T04:57:42.205037+0000","last_became_peered":"2023-10-09T04:57:42.205037+0000","last_unstale":"2023-10-09T04:58:39.234730+0000","last_undegraded":"2023-10-09T04:57:42.201899+0000","last_fullsized":"2023-10-09T04:57:42.201289+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":331,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.236505+0000","last_change":"2023-10-09T04:57:42.213719+0000","last_active":"2023-10-09T04:58:39.236505+0000","last_peered":"2023-10-09T04:58:39.236505+0000","last_clean":"2023-10-09T04:57:27.237200+0000","last_became_active":"2023-10-09T04:57:42.213719+0000","last_became_peered":"2023-10-09T04:57:42.213719+0000","last_unstale":"2023-10-09T04:58:39.236505+0000","last_undegraded":"2023-10-09T04:58:39.236505+0000","last_fullsized":"2023-10-09T04:57:42.202558+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":274,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233883+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T04:58:39.233883+0000","last_peered":"2023-10-09T04:58:39.233883+0000","last_clean":"2023-10-09T04:58:39.233883+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T04:58:39.233883+0000","last_undegraded":"2023-10-09T04:58:39.233883+0000","last_fullsized":"2023-10-09T04:58:39.233883+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":349,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.403295+0000","last_change":"2023-10-09T04:56:46.352313+0000","last_active":"2023-10-09T04:58:39.403295+0000","last_peered":"2023-10-09T04:58:39.403295+0000","last_clean":"2023-10-09T04:58:39.403295+0000","last_became_active":"2023-10-09T04:56:46.351505+0000","last_became_peered":"2023-10-09T04:56:46.351505+0000","last_unstale":"2023-10-09T04:58:39.403295+0000","last_undegraded":"2023-10-09T04:58:39.403295+0000","last_fullsized":"2023-10-09T04:58:39.403295+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:01.025392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":292,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.403418+0000","last_change":"2023-10-09T04:56:46.428588+0000","last_active":"2023-10-09T04:58:39.403418+0000","last_peered":"2023-10-09T04:58:39.403418+0000","last_clean":"2023-10-09T04:58:39.403418+0000","last_became_active":"2023-10-09T04:56:46.363039+0000","last_became_peered":"2023-10-09T04:56:46.363039+0000","last_unstale":"2023-10-09T04:58:39.403418+0000","last_undegraded":"2023-10-09T04:58:39.403418+0000","last_fullsized":"2023-10-09T04:58:39.403418+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:16.759915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810889+0000","last_change":"2023-10-09T04:56:46.359130+0000","last_active":"2023-10-09T04:58:39.810889+0000","last_peered":"2023-10-09T04:58:39.810889+0000","last_clean":"2023-10-09T04:58:39.810889+0000","last_became_active":"2023-10-09T04:56:46.358781+0000","last_became_peered":"2023-10-09T04:56:46.358781+0000","last_unstale":"2023-10-09T04:58:39.810889+0000","last_undegraded":"2023-10-09T04:58:39.810889+0000","last_fullsized":"2023-10-09T04:58:39.810889+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":326,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.236493+0000","last_change":"2023-10-09T04:56:46.360890+0000","last_active":"2023-10-09T04:58:39.236493+0000","last_peered":"2023-10-09T04:58:39.236493+0000","last_clean":"2023-10-09T04:58:39.236493+0000","last_became_active":"2023-10-09T04:56:46.360526+0000","last_became_peered":"2023-10-09T04:56:46.360526+0000","last_unstale":"2023-10-09T04:58:39.236493+0000","last_undegraded":"2023-10-09T04:58:39.236493+0000","last_fullsized":"2023-10-09T04:58:39.236493+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810165+0000","last_change":"2023-10-09T04:56:46.366317+0000","last_active":"2023-10-09T04:58:39.810165+0000","last_peered":"2023-10-09T04:58:39.810165+0000","last_clean":"2023-10-09T04:58:39.810165+0000","last_became_active":"2023-10-09T04:56:46.363159+0000","last_became_peered":"2023-10-09T04:56:46.363159+0000","last_unstale":"2023-10-09T04:58:39.810165+0000","last_undegraded":"2023-10-09T04:58:39.810165+0000","last_fullsized":"2023-10-09T04:58:39.810165+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":44,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.808326+0000","last_change":"2023-10-09T04:57:42.225587+0000","last_active":"2023-10-09T04:58:39.808326+0000","last_peered":"2023-10-09T04:58:39.808326+0000","last_clean":"2023-10-09T04:57:27.234007+0000","last_became_active":"2023-10-09T04:57:42.225587+0000","last_became_peered":"2023-10-09T04:57:42.225587+0000","last_unstale":"2023-10-09T04:58:39.808326+0000","last_undegraded":"2023-10-09T04:58:39.808326+0000","last_fullsized":"2023-10-09T04:57:42.210298+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810576+0000","last_change":"2023-10-09T04:56:46.365379+0000","last_active":"2023-10-09T04:58:39.810576+0000","last_peered":"2023-10-09T04:58:39.810576+0000","last_clean":"2023-10-09T04:58:39.810576+0000","last_became_active":"2023-10-09T04:56:46.365000+0000","last_became_peered":"2023-10-09T04:56:46.365000+0000","last_unstale":"2023-10-09T04:58:39.810576+0000","last_undegraded":"2023-10-09T04:58:39.810576+0000","last_fullsized":"2023-10-09T04:58:39.810576+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":326,"reported_epoch":341,"state":"active+undersized","last_fresh":"2023-10-09T04:58:41.153426+0000","last_change":"2023-10-09T04:57:42.200637+0000","last_active":"2023-10-09T04:58:41.153426+0000","last_peered":"2023-10-09T04:58:41.153426+0000","last_clean":"2023-10-09T04:57:27.226593+0000","last_became_active":"2023-10-09T04:57:42.200637+0000","last_became_peered":"2023-10-09T04:57:42.200637+0000","last_unstale":"2023-10-09T04:58:41.153426+0000","last_undegraded":"2023-10-09T04:58:41.153426+0000","last_fullsized":"2023-10-09T04:57:42.198516+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:02.301549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810067+0000","last_change":"2023-10-09T04:56:46.352531+0000","last_active":"2023-10-09T04:58:39.810067+0000","last_peered":"2023-10-09T04:58:39.810067+0000","last_clean":"2023-10-09T04:58:39.810067+0000","last_became_active":"2023-10-09T04:56:46.351012+0000","last_became_peered":"2023-10-09T04:56:46.351012+0000","last_unstale":"2023-10-09T04:58:39.810067+0000","last_undegraded":"2023-10-09T04:58:39.810067+0000","last_fullsized":"2023-10-09T04:58:39.810067+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'111","reported_seq":513,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.403315+0000","last_change":"2023-10-09T04:57:42.211943+0000","last_active":"2023-10-09T04:58:39.403315+0000","last_peered":"2023-10-09T04:58:39.403315+0000","last_clean":"2023-10-09T04:57:27.245755+0000","last_became_active":"2023-10-09T04:57:42.211943+0000","last_became_peered":"2023-10-09T04:57:42.211943+0000","last_unstale":"2023-10-09T04:58:39.403315+0000","last_undegraded":"2023-10-09T04:57:42.196701+0000","last_fullsized":"2023-10-09T04:57:42.196564+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:15.897809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":543,"num_read_kb":1002,"num_write":114,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.234163+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T04:58:39.234163+0000","last_peered":"2023-10-09T04:58:39.234163+0000","last_clean":"2023-10-09T04:58:39.234163+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T04:58:39.234163+0000","last_undegraded":"2023-10-09T04:58:39.234163+0000","last_fullsized":"2023-10-09T04:58:39.234163+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":284,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233635+0000","last_change":"2023-10-09T04:56:46.343490+0000","last_active":"2023-10-09T04:58:39.233635+0000","last_peered":"2023-10-09T04:58:39.233635+0000","last_clean":"2023-10-09T04:58:39.233635+0000","last_became_active":"2023-10-09T04:56:46.343094+0000","last_became_peered":"2023-10-09T04:56:46.343094+0000","last_unstale":"2023-10-09T04:58:39.233635+0000","last_undegraded":"2023-10-09T04:58:39.233635+0000","last_fullsized":"2023-10-09T04:58:39.233635+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'70","reported_seq":803,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233653+0000","last_change":"2023-10-09T04:34:36.646555+0000","last_active":"2023-10-09T04:58:39.233653+0000","last_peered":"2023-10-09T04:58:39.233653+0000","last_clean":"2023-10-09T04:58:39.233653+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T04:58:39.233653+0000","last_undegraded":"2023-10-09T04:58:39.233653+0000","last_fullsized":"2023-10-09T04:58:39.233653+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":736,"num_read_kb":3011,"num_write":53,"num_write_kb":2216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810145+0000","last_change":"2023-10-09T04:56:46.360770+0000","last_active":"2023-10-09T04:58:39.810145+0000","last_peered":"2023-10-09T04:58:39.810145+0000","last_clean":"2023-10-09T04:58:39.810145+0000","last_became_active":"2023-10-09T04:56:46.360413+0000","last_became_peered":"2023-10-09T04:56:46.360413+0000","last_unstale":"2023-10-09T04:58:39.810145+0000","last_undegraded":"2023-10-09T04:58:39.810145+0000","last_fullsized":"2023-10-09T04:58:39.810145+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":294,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.224031+0000","last_change":"2023-10-09T04:57:17.362172+0000","last_active":"2023-10-09T04:58:39.224031+0000","last_peered":"2023-10-09T04:58:39.224031+0000","last_clean":"2023-10-09T04:58:39.224031+0000","last_became_active":"2023-10-09T04:57:17.361793+0000","last_became_peered":"2023-10-09T04:57:17.361793+0000","last_unstale":"2023-10-09T04:58:39.224031+0000","last_undegraded":"2023-10-09T04:58:39.224031+0000","last_fullsized":"2023-10-09T04:58:39.224031+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:18.548269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'264","reported_seq":1204,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.403337+0000","last_change":"2023-10-09T04:57:17.363194+0000","last_active":"2023-10-09T04:58:39.403337+0000","last_peered":"2023-10-09T04:58:39.403337+0000","last_clean":"2023-10-09T04:58:39.403337+0000","last_became_active":"2023-10-09T04:57:17.361372+0000","last_became_peered":"2023-10-09T04:57:17.361372+0000","last_unstale":"2023-10-09T04:58:39.403337+0000","last_undegraded":"2023-10-09T04:58:39.403337+0000","last_fullsized":"2023-10-09T04:58:39.403337+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:52.123105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2152,"num_read_kb":1798,"num_write":218,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":245,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.236744+0000","last_change":"2023-10-09T04:57:17.372505+0000","last_active":"2023-10-09T04:58:39.236744+0000","last_peered":"2023-10-09T04:58:39.236744+0000","last_clean":"2023-10-09T04:58:39.236744+0000","last_became_active":"2023-10-09T04:57:17.372195+0000","last_became_peered":"2023-10-09T04:57:17.372195+0000","last_unstale":"2023-10-09T04:58:39.236744+0000","last_undegraded":"2023-10-09T04:58:39.236744+0000","last_fullsized":"2023-10-09T04:58:39.236744+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":211,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.234593+0000","last_change":"2023-10-09T04:50:44.051844+0000","last_active":"2023-10-09T04:58:39.234593+0000","last_peered":"2023-10-09T04:58:39.234593+0000","last_clean":"2023-10-09T04:58:39.234593+0000","last_became_active":"2023-10-09T04:50:44.051376+0000","last_became_peered":"2023-10-09T04:50:44.051376+0000","last_unstale":"2023-10-09T04:58:39.234593+0000","last_undegraded":"2023-10-09T04:58:39.234593+0000","last_fullsized":"2023-10-09T04:58:39.234593+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'177","reported_seq":872,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.234115+0000","last_change":"2023-10-09T04:57:42.207711+0000","last_active":"2023-10-09T04:58:39.234115+0000","last_peered":"2023-10-09T04:58:39.234115+0000","last_clean":"2023-10-09T04:57:27.229687+0000","last_became_active":"2023-10-09T04:57:42.207711+0000","last_became_peered":"2023-10-09T04:57:42.207711+0000","last_unstale":"2023-10-09T04:58:39.234115+0000","last_undegraded":"2023-10-09T04:57:42.201437+0000","last_fullsized":"2023-10-09T04:57:42.201251+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":177,"log_dups_size":0,"ondisk_log_size":177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":769,"num_read_kb":1697,"num_write":166,"num_write_kb":5218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":312,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.402834+0000","last_change":"2023-10-09T04:57:42.208959+0000","last_active":"2023-10-09T04:58:39.402834+0000","last_peered":"2023-10-09T04:58:39.402834+0000","last_clean":"2023-10-09T04:17:29.322482+0000","last_became_active":"2023-10-09T04:57:42.208959+0000","last_became_peered":"2023-10-09T04:57:42.208959+0000","last_unstale":"2023-10-09T04:58:39.402834+0000","last_undegraded":"2023-10-09T04:57:42.193856+0000","last_fullsized":"2023-10-09T04:57:42.193667+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:58.999891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233526+0000","last_change":"2023-10-09T04:34:36.646647+0000","last_active":"2023-10-09T04:58:39.233526+0000","last_peered":"2023-10-09T04:58:39.233526+0000","last_clean":"2023-10-09T04:58:39.233526+0000","last_became_active":"2023-10-09T04:34:36.643938+0000","last_became_peered":"2023-10-09T04:34:36.643938+0000","last_unstale":"2023-10-09T04:58:39.233526+0000","last_undegraded":"2023-10-09T04:58:39.233526+0000","last_fullsized":"2023-10-09T04:58:39.233526+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.237948+0000","last_change":"2023-10-09T04:57:42.214118+0000","last_active":"2023-10-09T04:58:39.237948+0000","last_peered":"2023-10-09T04:58:39.237948+0000","last_clean":"2023-10-09T04:57:27.237609+0000","last_became_active":"2023-10-09T04:57:42.214118+0000","last_became_peered":"2023-10-09T04:57:42.214118+0000","last_unstale":"2023-10-09T04:58:39.237948+0000","last_undegraded":"2023-10-09T04:58:39.237948+0000","last_fullsized":"2023-10-09T04:57:42.203522+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"77'122","reported_seq":666,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.403140+0000","last_change":"2023-10-09T04:57:42.203165+0000","last_active":"2023-10-09T04:58:39.403140+0000","last_peered":"2023-10-09T04:58:39.403140+0000","last_clean":"2023-10-09T04:13:52.721951+0000","last_became_active":"2023-10-09T04:57:42.203165+0000","last_became_peered":"2023-10-09T04:57:42.203165+0000","last_unstale":"2023-10-09T04:58:39.403140+0000","last_undegraded":"2023-10-09T04:58:39.403140+0000","last_fullsized":"2023-10-09T04:57:42.194520+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:37.187248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":966,"num_read_kb":862,"num_write":82,"num_write_kb":45,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":369,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.404248+0000","last_change":"2023-10-09T04:56:46.362929+0000","last_active":"2023-10-09T04:58:39.404248+0000","last_peered":"2023-10-09T04:58:39.404248+0000","last_clean":"2023-10-09T04:58:39.404248+0000","last_became_active":"2023-10-09T04:56:46.362453+0000","last_became_peered":"2023-10-09T04:56:46.362453+0000","last_unstale":"2023-10-09T04:58:39.404248+0000","last_undegraded":"2023-10-09T04:58:39.404248+0000","last_fullsized":"2023-10-09T04:58:39.404248+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:34.707239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":333,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.404096+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T04:58:39.404096+0000","last_peered":"2023-10-09T04:58:39.404096+0000","last_clean":"2023-10-09T04:58:39.404096+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T04:58:39.404096+0000","last_undegraded":"2023-10-09T04:58:39.404096+0000","last_fullsized":"2023-10-09T04:58:39.404096+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'109","reported_seq":316,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.403568+0000","last_change":"2023-10-09T04:57:42.217812+0000","last_active":"2023-10-09T04:58:39.403568+0000","last_peered":"2023-10-09T04:58:39.403568+0000","last_clean":"2023-10-09T04:13:52.132199+0000","last_became_active":"2023-10-09T04:57:42.217812+0000","last_became_peered":"2023-10-09T04:57:42.217812+0000","last_unstale":"2023-10-09T04:58:39.403568+0000","last_undegraded":"2023-10-09T04:58:39.403568+0000","last_fullsized":"2023-10-09T04:57:42.195843+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:48.901904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":392,"num_read_kb":1490,"num_write":109,"num_write_kb":2183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":245,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.234193+0000","last_change":"2023-10-09T04:57:17.362182+0000","last_active":"2023-10-09T04:58:39.234193+0000","last_peered":"2023-10-09T04:58:39.234193+0000","last_clean":"2023-10-09T04:58:39.234193+0000","last_became_active":"2023-10-09T04:57:17.361737+0000","last_became_peered":"2023-10-09T04:57:17.361737+0000","last_unstale":"2023-10-09T04:58:39.234193+0000","last_undegraded":"2023-10-09T04:58:39.234193+0000","last_fullsized":"2023-10-09T04:58:39.234193+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":328,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.404160+0000","last_change":"2023-10-09T04:57:42.206408+0000","last_active":"2023-10-09T04:58:39.404160+0000","last_peered":"2023-10-09T04:58:39.404160+0000","last_clean":"2023-10-09T04:57:27.245433+0000","last_became_active":"2023-10-09T04:57:42.206408+0000","last_became_peered":"2023-10-09T04:57:42.206408+0000","last_unstale":"2023-10-09T04:58:39.404160+0000","last_undegraded":"2023-10-09T04:58:39.404160+0000","last_fullsized":"2023-10-09T04:57:42.195249+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:16.186404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":57,"reported_epoch":340,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:58:39.404998+0000","last_change":"2023-10-09T04:57:42.203123+0000","last_active":"2023-10-09T04:58:39.404998+0000","last_peered":"2023-10-09T04:58:39.404998+0000","last_clean":"2023-10-09T04:40:15.076963+0000","last_became_active":"2023-10-09T04:57:42.203123+0000","last_became_peered":"2023-10-09T04:57:42.203123+0000","last_unstale":"2023-10-09T04:58:39.404998+0000","last_undegraded":"2023-10-09T04:57:42.194690+0000","last_fullsized":"2023-10-09T04:57:42.194598+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:15.006280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"341'92","reported_seq":432,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T04:58:40.660237+0000","last_change":"2023-10-09T04:56:46.571033+0000","last_active":"2023-10-09T04:58:40.660237+0000","last_peered":"2023-10-09T04:58:40.660237+0000","last_clean":"2023-10-09T04:58:40.660237+0000","last_became_active":"2023-10-09T04:56:46.365646+0000","last_became_peered":"2023-10-09T04:56:46.365646+0000","last_unstale":"2023-10-09T04:58:40.660237+0000","last_undegraded":"2023-10-09T04:58:40.660237+0000","last_fullsized":"2023-10-09T04:58:40.660237+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":373,"num_read_kb":2018,"num_write":69,"num_write_kb":1409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.402791+0000","last_change":"2023-10-09T04:57:42.203449+0000","last_active":"2023-10-09T04:58:39.402791+0000","last_peered":"2023-10-09T04:58:39.402791+0000","last_clean":"2023-10-09T04:40:15.042307+0000","last_became_active":"2023-10-09T04:57:42.203449+0000","last_became_peered":"2023-10-09T04:57:42.203449+0000","last_unstale":"2023-10-09T04:58:39.402791+0000","last_undegraded":"2023-10-09T04:58:39.402791+0000","last_fullsized":"2023-10-09T04:57:42.193626+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:35.031524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'32","reported_seq":427,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.238076+0000","last_change":"2023-10-09T04:57:42.207841+0000","last_active":"2023-10-09T04:58:39.238076+0000","last_peered":"2023-10-09T04:58:39.238076+0000","last_clean":"2023-10-09T04:57:27.237729+0000","last_became_active":"2023-10-09T04:57:42.207841+0000","last_became_peered":"2023-10-09T04:57:42.207841+0000","last_unstale":"2023-10-09T04:58:39.238076+0000","last_undegraded":"2023-10-09T04:58:39.238076+0000","last_fullsized":"2023-10-09T04:57:42.204198+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":345,"num_write":35,"num_write_kb":1211,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":333,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.404212+0000","last_change":"2023-10-09T04:56:46.422892+0000","last_active":"2023-10-09T04:58:39.404212+0000","last_peered":"2023-10-09T04:58:39.404212+0000","last_clean":"2023-10-09T04:58:39.404212+0000","last_became_active":"2023-10-09T04:56:46.363249+0000","last_became_peered":"2023-10-09T04:56:46.363249+0000","last_unstale":"2023-10-09T04:58:39.404212+0000","last_undegraded":"2023-10-09T04:58:39.404212+0000","last_fullsized":"2023-10-09T04:58:39.404212+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:48.389583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.808892+0000","last_change":"2023-10-09T04:56:46.358968+0000","last_active":"2023-10-09T04:58:39.808892+0000","last_peered":"2023-10-09T04:58:39.808892+0000","last_clean":"2023-10-09T04:58:39.808892+0000","last_became_active":"2023-10-09T04:56:46.358613+0000","last_became_peered":"2023-10-09T04:56:46.358613+0000","last_unstale":"2023-10-09T04:58:39.808892+0000","last_undegraded":"2023-10-09T04:58:39.808892+0000","last_fullsized":"2023-10-09T04:58:39.808892+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":332,"reported_epoch":340,"state":"active+undersized","last_fresh":"2023-10-09T04:58:39.403568+0000","last_change":"2023-10-09T04:57:42.217817+0000","last_active":"2023-10-09T04:58:39.403568+0000","last_peered":"2023-10-09T04:58:39.403568+0000","last_clean":"2023-10-09T04:57:27.245393+0000","last_became_active":"2023-10-09T04:57:42.217817+0000","last_became_peered":"2023-10-09T04:57:42.217817+0000","last_unstale":"2023-10-09T04:58:39.403568+0000","last_undegraded":"2023-10-09T04:58:39.403568+0000","last_fullsized":"2023-10-09T04:57:42.194898+0000","mapping_epoch":321,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:47.008560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233997+0000","last_change":"2023-10-09T04:57:17.359307+0000","last_active":"2023-10-09T04:58:39.233997+0000","last_peered":"2023-10-09T04:58:39.233997+0000","last_clean":"2023-10-09T04:58:39.233997+0000","last_became_active":"2023-10-09T04:57:17.358947+0000","last_became_peered":"2023-10-09T04:57:17.358947+0000","last_unstale":"2023-10-09T04:58:39.233997+0000","last_undegraded":"2023-10-09T04:58:39.233997+0000","last_fullsized":"2023-10-09T04:58:39.233997+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":292,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233956+0000","last_change":"2023-10-09T04:56:46.363704+0000","last_active":"2023-10-09T04:58:39.233956+0000","last_peered":"2023-10-09T04:58:39.233956+0000","last_clean":"2023-10-09T04:58:39.233956+0000","last_became_active":"2023-10-09T04:56:46.343047+0000","last_became_peered":"2023-10-09T04:56:46.343047+0000","last_unstale":"2023-10-09T04:58:39.233956+0000","last_undegraded":"2023-10-09T04:58:39.233956+0000","last_fullsized":"2023-10-09T04:58:39.233956+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.810516+0000","last_change":"2023-10-09T04:56:46.350221+0000","last_active":"2023-10-09T04:58:39.810516+0000","last_peered":"2023-10-09T04:58:39.810516+0000","last_clean":"2023-10-09T04:58:39.810516+0000","last_became_active":"2023-10-09T04:56:46.349461+0000","last_became_peered":"2023-10-09T04:56:46.349461+0000","last_unstale":"2023-10-09T04:58:39.810516+0000","last_undegraded":"2023-10-09T04:58:39.810516+0000","last_fullsized":"2023-10-09T04:58:39.810516+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":285,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.234099+0000","last_change":"2023-10-09T04:56:46.375959+0000","last_active":"2023-10-09T04:58:39.234099+0000","last_peered":"2023-10-09T04:58:39.234099+0000","last_clean":"2023-10-09T04:58:39.234099+0000","last_became_active":"2023-10-09T04:56:46.354426+0000","last_became_peered":"2023-10-09T04:56:46.354426+0000","last_unstale":"2023-10-09T04:58:39.234099+0000","last_undegraded":"2023-10-09T04:58:39.234099+0000","last_fullsized":"2023-10-09T04:58:39.234099+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.808996+0000","last_change":"2023-10-09T04:56:46.362257+0000","last_active":"2023-10-09T04:58:39.808996+0000","last_peered":"2023-10-09T04:58:39.808996+0000","last_clean":"2023-10-09T04:58:39.808996+0000","last_became_active":"2023-10-09T04:56:46.361599+0000","last_became_peered":"2023-10-09T04:56:46.361599+0000","last_unstale":"2023-10-09T04:58:39.808996+0000","last_undegraded":"2023-10-09T04:58:39.808996+0000","last_fullsized":"2023-10-09T04:58:39.808996+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":242,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.237974+0000","last_change":"2023-10-09T04:44:51.903194+0000","last_active":"2023-10-09T04:58:39.237974+0000","last_peered":"2023-10-09T04:58:39.237974+0000","last_clean":"2023-10-09T04:58:39.237974+0000","last_became_active":"2023-10-09T04:44:51.902510+0000","last_became_peered":"2023-10-09T04:44:51.902510+0000","last_unstale":"2023-10-09T04:58:39.237974+0000","last_undegraded":"2023-10-09T04:58:39.237974+0000","last_fullsized":"2023-10-09T04:58:39.237974+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":52,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.233470+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T04:58:39.233470+0000","last_peered":"2023-10-09T04:58:39.233470+0000","last_clean":"2023-10-09T04:58:39.233470+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T04:58:39.233470+0000","last_undegraded":"2023-10-09T04:58:39.233470+0000","last_fullsized":"2023-10-09T04:58:39.233470+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":338,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.405056+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T04:58:39.405056+0000","last_peered":"2023-10-09T04:58:39.405056+0000","last_clean":"2023-10-09T04:58:39.405056+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T04:58:39.405056+0000","last_undegraded":"2023-10-09T04:58:39.405056+0000","last_fullsized":"2023-10-09T04:58:39.405056+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T04:58:39.808230+0000","last_change":"2023-10-09T04:56:46.359415+0000","last_active":"2023-10-09T04:58:39.808230+0000","last_peered":"2023-10-09T04:58:39.808230+0000","last_clean":"2023-10-09T04:58:39.808230+0000","last_became_active":"2023-10-09T04:56:46.358569+0000","last_became_peered":"2023-10-09T04:56:46.358569+0000","last_unstale":"2023-10-09T04:58:39.808230+0000","last_undegraded":"2023-10-09T04:58:39.808230+0000","last_fullsized":"2023-10-09T04:58:39.808230+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":85,"acting":85,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":9650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42323,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":83,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5993,"num_read_kb":12223,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6416,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430798,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1164940,"kb_used_data":261252,"kb_used_omap":0,"kb_used_meta":903616,"kb_avail":92580212,"statfs":{"total":95995035648,"available":94802137088,"internally_reserved":0,"allocated":267522048,"data_stored":266915443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925302784},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:33 2023","interfaces":[{"interface":"back","average":{"1min":8.1340000000000003,"5min":10.19,"15min":5.0869999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.24199999999999999},"max":{"1min":59.469999999999999,"5min":422.35000000000002,"15min":422.35000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":9.4480000000000004,"5min":10.569000000000001,"15min":5.117},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26500000000000001},"max":{"1min":59.966000000000001,"5min":423.267,"15min":423.267},"last":0.83599999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:57:06 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":1.7490000000000001,"15min":2.3439999999999999},"min":{"1min":0.309,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.201000000000001,"5min":46.222999999999999,"15min":58.816000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.7410000000000001,"15min":2.3410000000000002},"min":{"1min":0.435,"5min":0.30599999999999999,"15min":0.29699999999999999},"max":{"1min":12.647,"5min":45.536000000000001,"15min":58.840000000000003},"last":5.04}]},{"osd":2,"last update":"Mon Oct 9 04:57:18 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.621,"15min":1.6359999999999999},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.314},"max":{"1min":11.856999999999999,"5min":11.856999999999999,"15min":13.138999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":5.1980000000000004,"5min":2.0009999999999999,"15min":1.915},"min":{"1min":0.35899999999999999,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":58.798999999999999,"5min":58.798999999999999,"15min":58.798999999999999},"last":1.0189999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:57:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.399,"15min":2.0499999999999998},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.26700000000000002},"max":{"1min":6.1470000000000002,"5min":8.6449999999999996,"15min":102.02800000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.6990000000000001,"15min":2.0649999999999999},"min":{"1min":0.437,"5min":0.36899999999999999,"15min":0.32100000000000001},"max":{"1min":6.1600000000000001,"5min":13.195,"15min":101.973},"last":0.56000000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.82599999999999996,"15min":1.2629999999999999},"min":{"1min":0.42199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.3700000000000001,"5min":5.5099999999999998,"15min":11.933999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.71399999999999997,"15min":1.298},"min":{"1min":0.36599999999999999,"5min":0.32600000000000001,"15min":0.308},"max":{"1min":1.5349999999999999,"5min":2.5329999999999999,"15min":8.452},"last":1.0389999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:57:21 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.095,"15min":1.5129999999999999},"min":{"1min":0.43099999999999999,"5min":0.34599999999999997,"15min":0.31900000000000001},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":11.268000000000001},"last":6.1689999999999996},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.95699999999999996,"15min":1.6240000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.242,"5min":5.9889999999999999,"15min":15.138999999999999},"last":1.7929999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.3859999999999999,"15min":1.407},"min":{"1min":0.41899999999999998,"5min":0.38800000000000001,"15min":0.36899999999999999},"max":{"1min":3.8450000000000002,"5min":27.059000000000001,"15min":27.059000000000001},"last":1.248},{"interface":"front","average":{"1min":1.173,"5min":1.6080000000000001,"15min":1.6020000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9320000000000004,"5min":27.698,"15min":27.698},"last":0.59499999999999997}]}]},{"osd":6,"up_from":146,"seq":627065225513,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617908,"kb_used_data":3564,"kb_used_omap":5,"kb_used_meta":614330,"kb_avail":93127244,"statfs":{"total":95995035648,"available":95362297856,"internally_reserved":0,"allocated":3649536,"data_stored":3037634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":629074528},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":3.516,"15min":3.5230000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.8460000000000001,"5min":81.287999999999997,"15min":214.82300000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.5009999999999999,"5min":3.3439999999999999,"15min":3.343},"min":{"1min":0.36699999999999999,"5min":0.26400000000000001,"15min":0.254},"max":{"1min":8.1199999999999992,"5min":81.168999999999997,"15min":213.34700000000001},"last":0.875}]},{"osd":1,"last update":"Mon Oct 9 04:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.359,"15min":2.5910000000000002},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":2.7879999999999998,"5min":9.9570000000000007,"15min":76.081000000000003},"last":0.69199999999999995},{"interface":"front","average":{"1min":3.5350000000000001,"5min":1.8959999999999999,"15min":2.4750000000000001},"min":{"1min":0.379,"5min":0.371,"15min":0.28299999999999997},"max":{"1min":18.539999999999999,"5min":18.539999999999999,"15min":75.546000000000006},"last":1.038}]},{"osd":2,"last update":"Mon Oct 9 04:58:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":3.7480000000000002,"15min":2.2469999999999999},"min":{"1min":0.42399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.422999999999998,"5min":97.284999999999997,"15min":97.284999999999997},"last":1.165},{"interface":"front","average":{"1min":1.145,"5min":3.3010000000000002,"15min":1.923},"min":{"1min":0.47099999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1310000000000002,"5min":97.069000000000003,"15min":97.069000000000003},"last":0.95999999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.8859999999999999,"15min":1.6759999999999999},"min":{"1min":0.434,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":5.8289999999999997,"5min":12.050000000000001,"15min":12.050000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.1110000000000002,"15min":1.7110000000000001},"min":{"1min":0.502,"5min":0.37,"15min":0.32600000000000001},"max":{"1min":18.387,"5min":18.387,"15min":18.387},"last":1.1100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.85399999999999998,"15min":1.127},"min":{"1min":0.432,"5min":0.317,"15min":0.248},"max":{"1min":4.343,"5min":4.9139999999999997,"15min":13.699999999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.3939999999999999,"15min":1.4330000000000001},"min":{"1min":0.51000000000000001,"5min":0.34499999999999997,"15min":0.307},"max":{"1min":4.4610000000000003,"5min":10.166,"15min":10.428000000000001},"last":0.92600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.4350000000000001,"15min":1.429},"min":{"1min":0.46400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8730000000000002,"5min":29.077999999999999,"15min":29.077999999999999},"last":1.125},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.835,"15min":1.879},"min":{"1min":0.36899999999999999,"5min":0.34999999999999998,"15min":0.29599999999999999},"max":{"1min":7.3239999999999998,"5min":28.742000000000001,"15min":28.742000000000001},"last":0.85299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.3700000000000001,"15min":1.5820000000000001},"min":{"1min":0.46400000000000002,"5min":0.35099999999999998,"15min":0.33000000000000002},"max":{"1min":2.6539999999999999,"5min":10.526999999999999,"15min":16.527999999999999},"last":5.0259999999999998},{"interface":"front","average":{"1min":1.206,"5min":1.1879999999999999,"15min":1.4730000000000001},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":5.0010000000000003,"5min":5.976,"15min":15.366},"last":0.57499999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958961,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917012,"kb_used_data":3660,"kb_used_omap":6,"kb_used_meta":913337,"kb_avail":92828140,"statfs":{"total":95995035648,"available":95056015360,"internally_reserved":0,"allocated":3747840,"data_stored":2526488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":935257100},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.2570000000000001,"15min":2.169},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.183},"max":{"1min":9.5999999999999996,"5min":47.274000000000001,"15min":47.274000000000001},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.7889999999999999,"5min":2.1269999999999998,"15min":1.9850000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23400000000000001},"max":{"1min":6.3879999999999999,"5min":47.372999999999998,"15min":47.372999999999998},"last":0.746}]},{"osd":2,"last update":"Mon Oct 9 04:58:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":1.7070000000000001,"15min":1.7729999999999999},"min":{"1min":0.36299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.004,"5min":10.004,"15min":15.336},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1040000000000001,"15min":1.796},"min":{"1min":0.246,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.367000000000001,"5min":15.048,"15min":15.048},"last":0.58599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:58:05 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":2.4969999999999999,"15min":2.0750000000000002},"min":{"1min":0.27700000000000002,"5min":0.23300000000000001,"15min":0.20999999999999999},"max":{"1min":12.771000000000001,"5min":68.974000000000004,"15min":68.974000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":3.1440000000000001,"5min":2.694,"15min":2.129},"min":{"1min":0.34100000000000003,"5min":0.26000000000000001,"15min":0.20200000000000001},"max":{"1min":12.629,"5min":69.183000000000007,"15min":69.183000000000007},"last":0.78200000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:57:53 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.9199999999999999,"15min":1.72},"min":{"1min":0.308,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":10.143000000000001,"5min":10.398,"15min":13.362},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.72,"5min":1.5429999999999999,"15min":1.621},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.316},"max":{"1min":6.1959999999999997,"5min":9.6259999999999994,"15min":9.827},"last":1.0780000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:58:08 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.7709999999999999,"15min":1.748},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.438000000000001,"5min":10.438000000000001,"15min":14.494999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":2.6309999999999998,"5min":2.024,"15min":1.915},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.192,"5min":10.435,"15min":13.413},"last":0.96999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:57:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":1.8200000000000001,"15min":1.7689999999999999},"min":{"1min":0.53700000000000003,"5min":0.35599999999999998,"15min":0.32600000000000001},"max":{"1min":6.8869999999999996,"5min":18.295000000000002,"15min":18.295000000000002},"last":1.1830000000000001},{"interface":"front","average":{"1min":2.1629999999999998,"5min":2.484,"15min":2.052},"min":{"1min":0.58299999999999996,"5min":0.42299999999999999,"15min":0.40500000000000003},"max":{"1min":6.9290000000000003,"5min":68.581000000000003,"15min":68.581000000000003},"last":1.212}]},{"osd":7,"last update":"Mon Oct 9 04:56:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4540000000000002,"5min":1.8160000000000001,"15min":1.861},"min":{"1min":0.48999999999999999,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":9.0830000000000002,"5min":10.462,"15min":13.474},"last":7.6379999999999999},{"interface":"front","average":{"1min":1.411,"5min":1.6299999999999999,"15min":1.7250000000000001},"min":{"1min":0.41599999999999998,"5min":0.39500000000000002,"15min":0.36399999999999999},"max":{"1min":3.9740000000000002,"5min":9.5909999999999993,"15min":9.8079999999999998},"last":0.76900000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122487,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937796,"kb_used_data":4284,"kb_used_omap":9,"kb_used_meta":933494,"kb_avail":92807356,"statfs":{"total":95995035648,"available":95034732544,"internally_reserved":0,"allocated":4386816,"data_stored":3126842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":955898353},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.7789999999999999,"15min":1.734},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.18099999999999999},"max":{"1min":9.1120000000000001,"5min":13.468999999999999,"15min":62.396000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.173,"5min":1.522,"15min":1.9530000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":5.0650000000000004,"5min":7.7149999999999999,"15min":62.496000000000002},"last":0.55800000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":1.744,"15min":1.671},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":7.5030000000000001,"5min":13.493,"15min":17.109000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.9410000000000001,"15min":1.946},"min":{"1min":0.36399999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.500999999999999,"5min":10.500999999999999,"15min":12.57},"last":0.503}]},{"osd":3,"last update":"Mon Oct 9 04:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":1.9830000000000001,"15min":1.6020000000000001},"min":{"1min":0.32400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1489999999999991,"5min":13.545999999999999,"15min":13.545999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.8540000000000001,"5min":2.1339999999999999,"15min":1.9430000000000001},"min":{"1min":0.39400000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.0690000000000008,"5min":13.544,"15min":13.544},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":2.202,"15min":1.8680000000000001},"min":{"1min":0.501,"5min":0.33800000000000002,"15min":0.27500000000000002},"max":{"1min":9.1370000000000005,"5min":14.996,"15min":16.84},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.7569999999999999,"15min":1.546},"min":{"1min":0.60899999999999999,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":9.1750000000000007,"5min":13.417,"15min":39.957999999999998},"last":0.97899999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.8500000000000001,"15min":1.7470000000000001},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.621,"5min":10.621,"15min":10.621},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.9159999999999999,"15min":1.643},"min":{"1min":0.45000000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.0449999999999999,"5min":13.374000000000001,"15min":13.374000000000001},"last":0.71999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:57:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.181,"15min":1.5680000000000001},"min":{"1min":0.46100000000000002,"5min":0.38200000000000001,"15min":0.33500000000000002},"max":{"1min":7.8650000000000002,"5min":13.445,"15min":13.445},"last":0.90700000000000003},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.1800000000000002,"15min":1.8700000000000001},"min":{"1min":0.47499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":9.7550000000000008,"5min":13.477,"15min":13.477},"last":0.878}]},{"osd":7,"last update":"Mon Oct 9 04:57:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":1.821,"15min":2.0059999999999998},"min":{"1min":0.53800000000000003,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":9.8719999999999999,"5min":13.518000000000001,"15min":43.276000000000003},"last":5.5490000000000004},{"interface":"front","average":{"1min":1.8089999999999999,"5min":2.0339999999999998,"15min":2.085},"min":{"1min":0.43099999999999999,"5min":0.39300000000000002,"15min":0.35499999999999998},"max":{"1min":8.2110000000000003,"5min":14.93,"15min":46.064},"last":2.7480000000000002}]}]},{"osd":2,"up_from":234,"seq":1005022347432,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":327560,"kb_used_data":7808,"kb_used_omap":13,"kb_used_meta":319730,"kb_avail":93417592,"statfs":{"total":95995035648,"available":95659614208,"internally_reserved":0,"allocated":7995392,"data_stored":6727222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":327404415},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2650000000000001,"5min":5.3449999999999998,"15min":3.7269999999999999},"min":{"1min":0.20899999999999999,"5min":0.20000000000000001,"15min":0.186},"max":{"1min":10.266999999999999,"5min":283.30799999999999,"15min":283.30799999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":4.1689999999999996,"5min":6.0259999999999998,"15min":3.976},"min":{"1min":0.32400000000000001,"5min":0.223,"15min":0.17999999999999999},"max":{"1min":14.542,"5min":282.24099999999999,"15min":282.24099999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":1.9910000000000001,"15min":4.6779999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":9.1999999999999993,"5min":13.311999999999999,"15min":267.54000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.1870000000000001,"5min":2.081,"15min":4.5540000000000003},"min":{"1min":0.311,"5min":0.24399999999999999,"15min":0.22900000000000001},"max":{"1min":3.2629999999999999,"5min":10.177,"15min":268.12200000000001},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.2250000000000001,"15min":1.675},"min":{"1min":0.32800000000000001,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":3.6070000000000002,"5min":14.048,"15min":14.340999999999999},"last":0.309},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.97,"15min":1.9099999999999999},"min":{"1min":0.34399999999999997,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.2460000000000004,"5min":9.532,"15min":13.167999999999999},"last":0.377}]},{"osd":4,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.6779999999999999,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.4039999999999999,"5min":9.2210000000000001,"15min":10.051},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.0030000000000001,"5min":1.79,"15min":1.583},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.316},"max":{"1min":9.1669999999999998,"5min":14.063000000000001,"15min":14.063000000000001},"last":0.79600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.6459999999999999,"15min":1.5840000000000001},"min":{"1min":0.372,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.2970000000000002,"5min":8.8740000000000006,"15min":10.856999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.8700000000000001,"15min":1.946},"min":{"1min":0.39900000000000002,"5min":0.39200000000000002,"15min":0.316},"max":{"1min":7.2169999999999996,"5min":8.2940000000000005,"15min":12.295999999999999},"last":0.75600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":2.2040000000000002,"15min":1.827},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.0899999999999999,"5min":14.012,"15min":14.012},"last":0.68000000000000005},{"interface":"front","average":{"1min":2.202,"5min":2.1509999999999998,"15min":1.647},"min":{"1min":0.34799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":9.3140000000000001,"5min":9.3759999999999994,"15min":10.890000000000001},"last":0.60299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:57:13 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.7869999999999999,"15min":1.9510000000000001},"min":{"1min":0.432,"5min":0.38,"15min":0.38},"max":{"1min":8.3770000000000007,"5min":8.3770000000000007,"15min":14.161},"last":1.627},{"interface":"front","average":{"1min":3.198,"5min":1.7310000000000001,"15min":1.7370000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":14.095000000000001,"5min":14.095000000000001,"15min":14.095000000000001},"last":0.38600000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631907,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917524,"kb_used_data":3724,"kb_used_omap":5,"kb_used_meta":913786,"kb_avail":92827628,"statfs":{"total":95995035648,"available":95055491072,"internally_reserved":0,"allocated":3813376,"data_stored":2536246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":935717038},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:11 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":5.4829999999999997,"15min":3.9340000000000002},"min":{"1min":0.40799999999999997,"5min":0.21299999999999999,"15min":0.19500000000000001},"max":{"1min":9.0440000000000005,"5min":263.22199999999998,"15min":263.22199999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.4889999999999999,"5min":6.0330000000000004,"15min":4.1790000000000003},"min":{"1min":0.442,"5min":0.255,"15min":0.20000000000000001},"max":{"1min":12.337,"5min":261.74599999999998,"15min":261.74599999999998},"last":0.53300000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.4399999999999999,"15min":2.3620000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.21199999999999999},"max":{"1min":3.6749999999999998,"5min":18.207000000000001,"15min":143.34100000000001},"last":0.44},{"interface":"front","average":{"1min":1.25,"5min":1.972,"15min":2.7440000000000002},"min":{"1min":0.24399999999999999,"5min":0.23699999999999999,"15min":0.19900000000000001},"max":{"1min":5.8860000000000001,"5min":17.943999999999999,"15min":149.15700000000001},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.556,"15min":1.4850000000000001},"min":{"1min":0.307,"5min":0.25,"15min":0.25},"max":{"1min":3.706,"5min":16.620000000000001,"15min":16.620000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.9900000000000002,"15min":1.9750000000000001},"min":{"1min":0.372,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.218999999999999,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.58399999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:58:11 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.3899999999999999,"15min":1.3959999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.309},"max":{"1min":5.9139999999999997,"5min":8.9930000000000003,"15min":8.9930000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.71,"5min":2.0590000000000002,"15min":1.675},"min":{"1min":0.40600000000000003,"5min":0.39200000000000002,"15min":0.29199999999999998},"max":{"1min":5.8650000000000002,"5min":15.522,"15min":15.522},"last":0.81299999999999994}]},{"osd":5,"last update":"Mon Oct 9 04:58:17 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.508,"15min":2.1549999999999998},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.34799999999999998},"max":{"1min":5.8300000000000001,"5min":23.073,"15min":23.073},"last":0.745},{"interface":"front","average":{"1min":1.73,"5min":2.4969999999999999,"15min":1.97},"min":{"1min":0.45200000000000001,"5min":0.379,"15min":0.34200000000000003},"max":{"1min":5.9489999999999998,"5min":22.832000000000001,"15min":22.832000000000001},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:57:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.583,"15min":1.528},"min":{"1min":0.37,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.3869999999999996,"5min":9.5280000000000005,"15min":10.102},"last":0.89600000000000002},{"interface":"front","average":{"1min":2.1680000000000001,"5min":1.8080000000000001,"15min":1.5029999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35399999999999998},"max":{"1min":7.0010000000000003,"5min":8.907,"15min":8.907},"last":0.79600000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.5429999999999999,"15min":1.6870000000000001},"min":{"1min":0.35499999999999998,"5min":0.33800000000000002,"15min":0.33100000000000002},"max":{"1min":3.8130000000000002,"5min":9.5749999999999993,"15min":15.170999999999999},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.6759999999999999,"15min":1.663},"min":{"1min":0.42699999999999999,"5min":0.38,"15min":0.36699999999999999},"max":{"1min":4.617,"5min":9.5600000000000005,"15min":9.5600000000000005},"last":0.502}]}]},{"osd":4,"up_from":30,"seq":128849019509,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897588,"kb_used_data":2988,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":92847564,"statfs":{"total":95995035648,"available":95075905536,"internally_reserved":0,"allocated":3059712,"data_stored":2447323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.5309999999999999,"15min":1.9770000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26400000000000001},"max":{"1min":2.5670000000000002,"5min":10.614000000000001,"15min":52.106999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.6859999999999999,"15min":1.8919999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":5.6790000000000003,"5min":10.42,"15min":52.289999999999999},"last":0.69799999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:58:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.506,"15min":2.2999999999999998},"min":{"1min":0.317,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":15.374000000000001,"5min":15.374000000000001,"15min":81.114999999999995},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.9630000000000001,"5min":1.5289999999999999,"15min":2.52},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":15.419,"5min":15.419,"15min":80.819999999999993},"last":0.91000000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.175,"15min":1.407},"min":{"1min":0.309,"5min":0.309,"15min":0.29799999999999999},"max":{"1min":8.6310000000000002,"5min":8.6310000000000002,"15min":10.249000000000001},"last":0.80200000000000005},{"interface":"front","average":{"1min":2.2959999999999998,"5min":1.639,"15min":1.7629999999999999},"min":{"1min":0.35099999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":15.34,"5min":15.34,"15min":15.34},"last":0.74199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":1.839,"15min":1.488},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.29699999999999999},"max":{"1min":6.1600000000000001,"5min":32.362000000000002,"15min":32.362000000000002},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.627,"5min":1.98,"15min":1.6850000000000001},"min":{"1min":0.39500000000000002,"5min":0.35499999999999998,"15min":0.30299999999999999},"max":{"1min":8.3490000000000002,"5min":32.387,"15min":32.387},"last":0.98899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.214,"5min":1.1899999999999999,"15min":1.9650000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.251},"max":{"1min":15.113,"5min":15.113,"15min":78.885000000000005},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.1279999999999999,"15min":1.637},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":12.58,"5min":12.58,"15min":78.472999999999999},"last":1.2689999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.1870000000000001,"15min":1.2410000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":8.6999999999999993,"5min":8.6999999999999993,"15min":8.6999999999999993},"last":1.339},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.5600000000000001,"15min":1.528},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.5169999999999995,"5min":8.5169999999999995,"15min":9.3759999999999994},"last":1.069}]},{"osd":7,"last update":"Mon Oct 9 04:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.0600000000000001,"15min":1.5009999999999999},"min":{"1min":0.47199999999999998,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":2.8540000000000001,"5min":5.1059999999999999,"15min":7.4029999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.83399999999999996,"5min":1.0229999999999999,"15min":1.3620000000000001},"min":{"1min":0.505,"5min":0.46100000000000002,"15min":0.35099999999999998},"max":{"1min":1.7250000000000001,"5min":5.0049999999999999,"15min":8.9670000000000005},"last":0.76000000000000001}]}]},{"osd":5,"up_from":150,"seq":644245094691,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958868,"kb_used_data":7068,"kb_used_omap":6,"kb_used_meta":951737,"kb_avail":92786284,"statfs":{"total":95995035648,"available":95013154816,"internally_reserved":0,"allocated":7237632,"data_stored":6637075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":974578963},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:32 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":5.8029999999999999,"15min":4.1550000000000002},"min":{"1min":0.315,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":51.097000000000001,"5min":363.64499999999998,"15min":363.64499999999998},"last":0.77300000000000002},{"interface":"front","average":{"1min":3.6829999999999998,"5min":5.4820000000000002,"15min":4.3090000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":50.883000000000003,"5min":363.26900000000001,"15min":363.26900000000001},"last":0.69499999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.2250000000000001,"15min":2.895},"min":{"1min":0.379,"5min":0.308,"15min":0.28399999999999997},"max":{"1min":3.0310000000000001,"5min":4.9450000000000003,"15min":130.762},"last":1.05},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.359,"15min":3.149},"min":{"1min":0.41199999999999998,"5min":0.29499999999999998,"15min":0.26400000000000001},"max":{"1min":8.1929999999999996,"5min":8.1929999999999996,"15min":130.86199999999999},"last":0.81999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":3.1560000000000001,"15min":1.9730000000000001},"min":{"1min":0.313,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.7370000000000001,"5min":208.40100000000001,"15min":208.40100000000001},"last":1.167},{"interface":"front","average":{"1min":2.3639999999999999,"5min":3.4489999999999998,"15min":2.3300000000000001},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.705,"5min":208.078,"15min":208.078},"last":0.96799999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2929999999999999,"15min":2.3439999999999999},"min":{"1min":0.34300000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.8940000000000001,"5min":12.779,"15min":130.69200000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.621,"5min":1.6160000000000001,"15min":2.573},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.2739999999999991,"5min":11.715,"15min":130.53399999999999},"last":0.90300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.075,"15min":1.456},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":8.3010000000000002,"5min":8.3010000000000002,"15min":12.734},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.252,"5min":1.165,"15min":1.351},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7850000000000001,"5min":7.6760000000000002,"15min":10.709},"last":1.099}]},{"osd":6,"last update":"Mon Oct 9 04:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.1020000000000001,"15min":1.3169999999999999},"min":{"1min":0.40899999999999997,"5min":0.35599999999999998,"15min":0.33900000000000002},"max":{"1min":5.0220000000000002,"5min":5.0220000000000002,"15min":10.77},"last":0.997},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.27,"15min":1.575},"min":{"1min":0.37,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.25,"5min":9.1600000000000001,"15min":14.223000000000001},"last":1.129}]},{"osd":7,"last update":"Mon Oct 9 04:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.337,"15min":1.843},"min":{"1min":0.376,"5min":0.376,"15min":0.32200000000000001},"max":{"1min":5.7949999999999999,"5min":7.1200000000000001,"15min":12.706},"last":0.874},{"interface":"front","average":{"1min":1.907,"5min":1.268,"15min":1.696},"min":{"1min":0.438,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":5.9809999999999999,"5min":6.5300000000000002,"15min":21.797000000000001},"last":0.38900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:58:59.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T04:58:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.349 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:59.450 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.481 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:59.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img2 /tmp/rbd_import_export_244924/img3 2023-10-09T04:58:59.585 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:59.617 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:59.721 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:59.755 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:59.859 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:59.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:59.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:58:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:58:59] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T04:58:59.994 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.022 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:00.126 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.150 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:00.238 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd help export 2023-10-09T04:59:00.238 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -q export-format 2023-10-09T04:59:00.254 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:00.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:00 smithi153 bash[59565]: cluster 2023-10-09T04:58:58.949927+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v1505: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 157 KiB/s rd, 71 KiB/s wr, 27 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:00.268 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:00 smithi153 bash[59565]: audit 2023-10-09T04:58:59.309170+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:00.286 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:00.390 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:00.426 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:00 smithi082 bash[77063]: cluster 2023-10-09T04:58:58.949927+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v1505: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 157 KiB/s rd, 71 KiB/s wr, 27 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:00 smithi082 bash[77063]: audit 2023-10-09T04:58:59.309170+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:00 smithi082 bash[79733]: cluster 2023-10-09T04:58:58.949927+0000 mgr.y (mgr.34107) 1353 : cluster 0 pgmap v1505: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 157 KiB/s rd, 71 KiB/s wr, 27 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:00.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:00 smithi082 bash[79733]: audit 2023-10-09T04:58:59.309170+0000 mon.a (mon.0) 1947 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:00.530 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.560 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:00.604 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_244924/img bs=1k count=10 seek=100 2023-10-09T04:59:00.663 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:00.690 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:00.744 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records in 2023-10-09T04:59:00.744 INFO:tasks.workunit.client.0.smithi082.stderr:10+0 records out 2023-10-09T04:59:00.744 INFO:tasks.workunit.client.0.smithi082.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000137014 s, 74.7 MB/s 2023-10-09T04:59:00.745 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:59:00.793 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:00.814 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:00.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:00.820 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:00.820 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:00.823 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap 2023-10-09T04:59:00.918 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:00.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:01.058 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:01.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:01.218 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:01.234 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:59:01.243 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T04:59:01.310 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta list testimg 2023-10-09T04:59:01.321 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:01.348 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:01.366 INFO:tasks.workunit.client.0.smithi082.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T04:59:01.366 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:59:01.366 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:59:01.366 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 2023-10-09T04:59:01.367 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:01.422 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:01.427 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_244924/img_v2 testimg_import 2023-10-09T04:59:01.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.471 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:01.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:01.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:01.701 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:01.731 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:01.835 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.864 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:01.967 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.995 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:01.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:02.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:02.129 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:02.233 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:02.247 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:59:02.251 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:59:02.260 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:02.295 INFO:tasks.workunit.client.0.smithi082.stdout:rbd image 'testimg_import': 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: size 110 KiB in 1 objects 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: order 22 (4 MiB objects) 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: snapshot_count: 1 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: id: b408797ccaff 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: block_name_prefix: rbd_data.b408797ccaff 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: format: 2 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: op_features: 2023-10-09T04:59:02.296 INFO:tasks.workunit.client.0.smithi082.stdout: flags: 2023-10-09T04:59:02.297 INFO:tasks.workunit.client.0.smithi082.stdout: create_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.297 INFO:tasks.workunit.client.0.smithi082.stdout: access_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.297 INFO:tasks.workunit.client.0.smithi082.stdout: modify_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.301 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import@snap 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout:rbd image 'testimg_import': 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: size 110 KiB in 1 objects 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: order 22 (4 MiB objects) 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: snapshot_count: 1 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: id: b408797ccaff 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: block_name_prefix: rbd_data.b408797ccaff 2023-10-09T04:59:02.346 INFO:tasks.workunit.client.0.smithi082.stdout: format: 2 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: op_features: 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: flags: 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: create_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: access_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: modify_timestamp: Mon Oct 9 04:59:01 2023 2023-10-09T04:59:02.347 INFO:tasks.workunit.client.0.smithi082.stdout: protected: False 2023-10-09T04:59:02.351 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd image-meta list testimg_import 2023-10-09T04:59:02.364 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:02.376 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:02 smithi153 bash[59565]: cluster 2023-10-09T04:59:00.951344+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v1506: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 213 KiB/s rd, 88 KiB/s wr, 66 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:02.376 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:02 smithi153 bash[59565]: cluster 2023-10-09T04:59:01.215747+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.376 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:02 smithi153 bash[59565]: cluster 2023-10-09T04:59:01.219717+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 7 up, 8 in 2023-10-09T04:59:02.376 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:02 smithi153 bash[59565]: cluster 2023-10-09T04:59:01.228032+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.396 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:02.399 INFO:tasks.workunit.client.0.smithi082.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T04:59:02.400 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:59:02.400 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:59:02.400 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 2023-10-09T04:59:02.400 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T04:59:02.400 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:59:02.401 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:59:02.401 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T04:59:02.401 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:59:02.401 INFO:tasks.workunit.client.0.smithi082.stderr:Key Value 2023-10-09T04:59:02.401 INFO:tasks.workunit.client.0.smithi082.stderr:key1 value1 ] 2023-10-09T04:59:02.402 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg_import /tmp/rbd_import_export_244924/img_import 2023-10-09T04:59:02.443 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:59:02.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img_import 2023-10-09T04:59:02.448 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/img /tmp/rbd_import_export_244924/img_import 2023-10-09T04:59:02.449 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/img --format %b 2023-10-09T04:59:02.450 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=24 2023-10-09T04:59:02.450 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/img_import --format %b 2023-10-09T04:59:02.450 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=224 2023-10-09T04:59:02.451 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=200 2023-10-09T04:59:02.451 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=200 2023-10-09T04:59:02.451 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T04:59:02.451 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_244924/img_snap 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[77063]: cluster 2023-10-09T04:59:00.951344+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v1506: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 213 KiB/s rd, 88 KiB/s wr, 66 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[77063]: cluster 2023-10-09T04:59:01.215747+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[77063]: cluster 2023-10-09T04:59:01.219717+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 7 up, 8 in 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[77063]: cluster 2023-10-09T04:59:01.228032+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[79733]: cluster 2023-10-09T04:59:00.951344+0000 mgr.y (mgr.34107) 1354 : cluster 0 pgmap v1506: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 6.5 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 213 KiB/s rd, 88 KiB/s wr, 66 op/s; 18/90 objects degraded (20.000%) 2023-10-09T04:59:02.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[79733]: cluster 2023-10-09T04:59:01.215747+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[79733]: cluster 2023-10-09T04:59:01.219717+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 7 up, 8 in 2023-10-09T04:59:02.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:02 smithi082 bash[79733]: cluster 2023-10-09T04:59:01.228032+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:02.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:02.500 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:59:02.506 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_244924/img_snap_import 2023-10-09T04:59:02.527 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:02.552 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:59:02.558 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/img_snap /tmp/rbd_import_export_244924/img_snap_import 2023-10-09T04:59:02.558 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/img_snap /tmp/rbd_import_export_244924/img_snap_import 2023-10-09T04:59:02.559 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/img_snap --format %b 2023-10-09T04:59:02.560 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=224 2023-10-09T04:59:02.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/img_snap_import --format %b 2023-10-09T04:59:02.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=224 2023-10-09T04:59:02.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:02.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:02.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:59:02.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:02.563 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_import 2023-10-09T04:59:02.563 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_snap 2023-10-09T04:59:02.564 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_snap_import 2023-10-09T04:59:02.565 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T04:59:02.631 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:02.654 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:02.757 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:02.783 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:02.887 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:02.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:02.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:03.024 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:03.055 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:03.159 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.192 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:03.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[77063]: cluster 2023-10-09T04:59:01.414768+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:03.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[77063]: cluster 2023-10-09T04:59:02.223536+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[77063]: cluster 2023-10-09T04:59:02.223725+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[77063]: cluster 2023-10-09T04:59:02.226107+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 7 up, 8 in 2023-10-09T04:59:03.237 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[77063]: cluster 2023-10-09T04:59:02.229829+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[79733]: cluster 2023-10-09T04:59:01.414768+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:03.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[79733]: cluster 2023-10-09T04:59:02.223536+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[79733]: cluster 2023-10-09T04:59:02.223725+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[79733]: cluster 2023-10-09T04:59:02.226107+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 7 up, 8 in 2023-10-09T04:59:03.238 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:03 smithi082 bash[79733]: cluster 2023-10-09T04:59:02.229829+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.241 INFO:tasks.workunit.client.0.smithi082.stderr: Removing snap: 100% complete...done. 2023-10-09T04:59:03.256 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:59:03.298 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:03.321 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:03.393 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:03.397 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap rm testimg@snap 2023-10-09T04:59:03.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:03 smithi153 bash[59565]: cluster 2023-10-09T04:59:01.414768+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T04:59:03.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:03 smithi153 bash[59565]: cluster 2023-10-09T04:59:02.223536+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:03 smithi153 bash[59565]: cluster 2023-10-09T04:59:02.223725+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:03 smithi153 bash[59565]: cluster 2023-10-09T04:59:02.226107+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 7 up, 8 in 2023-10-09T04:59:03.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:03 smithi153 bash[59565]: cluster 2023-10-09T04:59:02.229829+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T04:59:03.426 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:03.452 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:03.557 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:03.589 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:03.693 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.725 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:03.829 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.856 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:03.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:03.927 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:03.927 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:03.959 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:03.990 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:03.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:04.003 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:59:04.027 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:59:04.052 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:59:04.077 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:59:04.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:04.104 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:59:04.107 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:04.116 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:04.134 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:59:04.156 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:59:04.156 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.156 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.156 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.157 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.157 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.157 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.157 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:59:04.157 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T04:59:04.221 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:04.246 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:04.253 INFO:tasks.workunit.client.0.smithi082.stderr: Removing snap: 100% complete...done. 2023-10-09T04:59:04.264 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm testimg 2023-10-09T04:59:04.350 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:04.372 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:04.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:04.405 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:04.410 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --order 20 /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:59:04.475 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:04.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:02.952193+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v1509: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 69 KiB/s wr, 120 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.223368+0000 mon.a (mon.0) 1950 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.233366+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.234992+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e344: 8 total, 7 up, 8 in 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.240365+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.240543+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: cluster 2023-10-09T04:59:03.242601+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[77063]: audit 2023-10-09T04:59:03.976236+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.82:0/818229553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:02.952193+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v1509: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 69 KiB/s wr, 120 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.223368+0000 mon.a (mon.0) 1950 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.233366+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.234992+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e344: 8 total, 7 up, 8 in 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.240365+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.240543+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: cluster 2023-10-09T04:59:03.242601+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:04 smithi082 bash[79733]: audit 2023-10-09T04:59:03.976236+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.82:0/818229553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:04.483 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:04.483 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:04.488 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:04.502 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:04.539 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:04.544 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_244924/img_v2 testimg_import 2023-10-09T04:59:04.604 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:04.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:02.952193+0000 mgr.y (mgr.34107) 1355 : cluster 0 pgmap v1509: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 215 KiB/s rd, 69 KiB/s wr, 120 op/s; 19/96 objects degraded (19.792%) 2023-10-09T04:59:04.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.223368+0000 mon.a (mon.0) 1950 : cluster 3 Health check update: Degraded data redundancy: 19/96 objects degraded (19.792%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:04.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.233366+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.234992+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e344: 8 total, 7 up, 8 in 2023-10-09T04:59:04.618 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.240365+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.618 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.240543+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.618 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: cluster 2023-10-09T04:59:03.242601+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T04:59:04.618 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:04 smithi153 bash[59565]: audit 2023-10-09T04:59:03.976236+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.82:0/818229553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:04.623 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:59:04.629 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd+ info testimg_import 2023-10-09T04:59:04.629 INFO:tasks.workunit.client.0.smithi082.stderr:grep order 2023-10-09T04:59:04.629 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $2} 2023-10-09T04:59:04.629 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep 20 2023-10-09T04:59:04.636 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:04.740 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:04.766 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:04.869 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:04.904 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:04.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:05.007 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.034 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:05.130 INFO:tasks.workunit.client.0.smithi082.stdout:20 2023-10-09T04:59:05.131 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:05.131 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:59:05.137 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.165 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:05.253 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is set 2023-10-09T04:59:05.268 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.269 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:05.272 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:59:05.275 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[77063]: cluster 2023-10-09T04:59:04.238085+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 7 up, 8 in 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[77063]: cluster 2023-10-09T04:59:04.240355+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[77063]: cluster 2023-10-09T04:59:04.245767+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[77063]: audit 2023-10-09T04:59:04.483993+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.82:0/3993746179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[77063]: audit 2023-10-09T04:59:04.484368+0000 mon.a (mon.0) 1953 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[79733]: cluster 2023-10-09T04:59:04.238085+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 7 up, 8 in 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[79733]: cluster 2023-10-09T04:59:04.240355+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[79733]: cluster 2023-10-09T04:59:04.245767+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.276 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[79733]: audit 2023-10-09T04:59:04.483993+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.82:0/3993746179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.277 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:05 smithi082 bash[79733]: audit 2023-10-09T04:59:04.484368+0000 mon.a (mon.0) 1953 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.289 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:05.393 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.405 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:05.409 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:59:05.414 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:05.469 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:05.473 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg 2023-10-09T04:59:05.473 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep features 2023-10-09T04:59:05.516 INFO:tasks.workunit.client.0.smithi082.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T04:59:05.516 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: 2023-10-09T04:59:05.516 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:05.517 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:05.539 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:05.560 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:05.564 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_244924/img_v2 testimg_import 2023-10-09T04:59:05.638 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:59:05.642 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:59:05.642 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep features 2023-10-09T04:59:05.642 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:05 smithi153 bash[59565]: cluster 2023-10-09T04:59:04.238085+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e345: 8 total, 7 up, 8 in 2023-10-09T04:59:05.654 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:05 smithi153 bash[59565]: cluster 2023-10-09T04:59:04.240355+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:05 smithi153 bash[59565]: cluster 2023-10-09T04:59:04.245767+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T04:59:05.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:05 smithi153 bash[59565]: audit 2023-10-09T04:59:04.483993+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.82:0/3993746179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:05 smithi153 bash[59565]: audit 2023-10-09T04:59:04.484368+0000 mon.a (mon.0) 1953 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:05.678 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:05.690 INFO:tasks.workunit.client.0.smithi082.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T04:59:05.690 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: 2023-10-09T04:59:05.691 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ features: layering 2023-10-09T04:59:05.691 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: != features: layering 2023-10-09T04:59:05.691 INFO:tasks.workunit.client.0.smithi082.stderr: op_features: ] 2023-10-09T04:59:05.691 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:05.691 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:59:05.783 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.799 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:05.803 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:59:05.810 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:05.914 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.919 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:05.923 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:59:05.946 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:05.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:05.994 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:05.997 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:06.050 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:06.072 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:06.174 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:06.195 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:06.198 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:06.200 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_244924/img_v2 testimg_import 2023-10-09T04:59:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:06.290 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:59:06.295 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import 2023-10-09T04:59:06.295 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep stripe unit 2023-10-09T04:59:06.295 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T04:59:06.302 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:06.329 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:06.344 INFO:tasks.workunit.client.0.smithi082.stdout: stripe unit: 4 KiB 2023-10-09T04:59:06.345 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbdgrep stripe count info 2023-10-09T04:59:06.345 INFO:tasks.workunit.client.0.smithi082.stderr: testimg_import 2023-10-09T04:59:06.345 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $3} 2023-10-09T04:59:06.346 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep 1000 2023-10-09T04:59:06.399 INFO:tasks.workunit.client.0.smithi082.stdout:1000 2023-10-09T04:59:06.399 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/img_v2 2023-10-09T04:59:06.432 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:06.455 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:06.464 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:04.952863+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v1512: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 12 KiB/s wr, 120 op/s; 20/99 objects degraded (20.202%) 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:05.235913+0000 mon.a (mon.0) 1954 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:05.244600+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: audit 2023-10-09T04:59:05.247189+0000 mon.a (mon.0) 1955 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:05.247299+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e346: 8 total, 7 up, 8 in 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:05.250344+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[77063]: cluster 2023-10-09T04:59:05.261848+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.557 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:04.952863+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v1512: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 12 KiB/s wr, 120 op/s; 20/99 objects degraded (20.202%) 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:05.235913+0000 mon.a (mon.0) 1954 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:05.244600+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: audit 2023-10-09T04:59:05.247189+0000 mon.a (mon.0) 1955 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:05.247299+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e346: 8 total, 7 up, 8 in 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:05.250344+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.569 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:06 smithi153 bash[59565]: cluster 2023-10-09T04:59:05.261848+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.587 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:06.617 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:59:06.623 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:59:06.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:06.712 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:06.728 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:04.952863+0000 mgr.y (mgr.34107) 1356 : cluster 0 pgmap v1512: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s rd, 12 KiB/s wr, 120 op/s; 20/99 objects degraded (20.202%) 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:05.235913+0000 mon.a (mon.0) 1954 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:05.244600+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: audit 2023-10-09T04:59:05.247189+0000 mon.a (mon.0) 1955 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:05.247299+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e346: 8 total, 7 up, 8 in 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:05.250344+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.729 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:06 smithi082 bash[79733]: cluster 2023-10-09T04:59:05.261848+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T04:59:06.745 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:59:06.748 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_244924/img testimg 2023-10-09T04:59:06.815 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.843 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:06.864 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:06.867 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap1 2023-10-09T04:59:06.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.977 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:06.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:07.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:07.112 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:07.215 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.231 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:59:07.242 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:07.267 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:59:07.276 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap create testimg@snap2 2023-10-09T04:59:07.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:07.366 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:07.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.489 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:07.593 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.620 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:07.723 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:07.754 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:07.857 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:07.888 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:07.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:07.993 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.023 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:08.126 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:59:08.146 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:59:08.158 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:08.259 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[79733]: cluster 2023-10-09T04:59:06.953962+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v1514: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 158 KiB/s rd, 13 KiB/s wr, 198 op/s; 21/102 objects degraded (20.588%) 2023-10-09T04:59:08.260 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[79733]: cluster 2023-10-09T04:59:07.248006+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.260 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[79733]: cluster 2023-10-09T04:59:07.250441+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.260 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[79733]: cluster 2023-10-09T04:59:07.252713+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e347: 8 total, 7 up, 8 in 2023-10-09T04:59:08.260 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[79733]: cluster 2023-10-09T04:59:07.259070+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.261 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:08.266 INFO:tasks.workunit.client.0.smithi082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:59:08.275 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap protect testimg@snap2 2023-10-09T04:59:08.289 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:08.327 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_244924/snap_protect 2023-10-09T04:59:08.385 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:08.390 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_244924/snap_protect testimg_import 2023-10-09T04:59:08.392 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.419 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:08.522 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:08.534 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:08 smithi153 bash[59565]: cluster 2023-10-09T04:59:06.953962+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v1514: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 158 KiB/s rd, 13 KiB/s wr, 198 op/s; 21/102 objects degraded (20.588%) 2023-10-09T04:59:08.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:08 smithi153 bash[59565]: cluster 2023-10-09T04:59:07.248006+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:08 smithi153 bash[59565]: cluster 2023-10-09T04:59:07.250441+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:08 smithi153 bash[59565]: cluster 2023-10-09T04:59:07.252713+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e347: 8 total, 7 up, 8 in 2023-10-09T04:59:08.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:08 smithi153 bash[59565]: cluster 2023-10-09T04:59:07.259070+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.554 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:08.658 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:08.666 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[77063]: cluster 2023-10-09T04:59:06.953962+0000 mgr.y (mgr.34107) 1357 : cluster 0 pgmap v1514: 73 pgs: 17 active+undersized, 13 active+undersized+degraded, 43 active+clean; 4.7 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 158 KiB/s rd, 13 KiB/s wr, 198 op/s; 21/102 objects degraded (20.588%) 2023-10-09T04:59:08.666 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[77063]: cluster 2023-10-09T04:59:07.248006+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.666 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[77063]: cluster 2023-10-09T04:59:07.250441+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.667 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[77063]: cluster 2023-10-09T04:59:07.252713+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e347: 8 total, 7 up, 8 in 2023-10-09T04:59:08.667 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:08 smithi082 bash[77063]: cluster 2023-10-09T04:59:07.259070+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T04:59:08.683 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:08.786 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.819 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:08.922 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:08.955 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:08.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:08.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:09.059 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:09.088 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:09.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:09.192 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:09.226 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:09.330 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:09.357 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:09.460 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:09.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:09 smithi153 bash[59565]: cluster 2023-10-09T04:59:08.251335+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:09 smithi153 bash[59565]: cluster 2023-10-09T04:59:08.255338+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e348: 8 total, 7 up, 8 in 2023-10-09T04:59:09.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:09 smithi153 bash[59565]: cluster 2023-10-09T04:59:08.259006+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.474 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:09 smithi153 bash[59565]: cluster 2023-10-09T04:59:08.955539+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v1517: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 274 KiB/s rd, 29 KiB/s wr, 364 op/s; 20/105 objects degraded (19.048%) 2023-10-09T04:59:09.496 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:09.600 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:09.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[79733]: cluster 2023-10-09T04:59:08.251335+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[79733]: cluster 2023-10-09T04:59:08.255338+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e348: 8 total, 7 up, 8 in 2023-10-09T04:59:09.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[79733]: cluster 2023-10-09T04:59:08.259006+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[79733]: cluster 2023-10-09T04:59:08.955539+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v1517: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 274 KiB/s rd, 29 KiB/s wr, 364 op/s; 20/105 objects degraded (19.048%) 2023-10-09T04:59:09.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[77063]: cluster 2023-10-09T04:59:08.251335+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[77063]: cluster 2023-10-09T04:59:08.255338+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e348: 8 total, 7 up, 8 in 2023-10-09T04:59:09.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[77063]: cluster 2023-10-09T04:59:08.259006+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:09.615 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[77063]: cluster 2023-10-09T04:59:08.955539+0000 mgr.y (mgr.34107) 1358 : cluster 0 pgmap v1517: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 274 KiB/s rd, 29 KiB/s wr, 364 op/s; 20/105 objects degraded (19.048%) 2023-10-09T04:59:09.633 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:09.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:09.737 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:09.770 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:09.874 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:09.886 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:09] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T04:59:09.903 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:09.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:10.006 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:10.035 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:10.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:10.147 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:59:10.169 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:10.274 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:10.296 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:10.307 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:59:10.312 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbd infogrep testimg_import@snap1 protected: False 2023-10-09T04:59:10.312 INFO:tasks.workunit.client.0.smithi082.stderr: 2023-10-09T04:59:10.364 INFO:tasks.workunit.client.0.smithi082.stdout: protected: False 2023-10-09T04:59:10.365 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info testimg_import@snap2 2023-10-09T04:59:10.365 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep protected: True 2023-10-09T04:59:10.399 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:10.408 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:08.372715+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:09.257193+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 20/105 objects degraded (19.048%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:09.267019+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:09.269752+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 7 up, 8 in 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:09.273413+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[77063]: cluster 2023-10-09T04:59:09.285427+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:08.372715+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:09.257193+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 20/105 objects degraded (19.048%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:10.409 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:09.267019+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:09.269752+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 7 up, 8 in 2023-10-09T04:59:10.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:09.273413+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.410 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:10 smithi082 bash[79733]: cluster 2023-10-09T04:59:09.285427+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.418 INFO:tasks.workunit.client.0.smithi082.stdout: protected: True 2023-10-09T04:59:10.418 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/snap_protect 2023-10-09T04:59:10.419 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T04:59:10.427 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:10.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:10.480 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T04:59:10.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:10.542 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:08.372715+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T04:59:10.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:09.257193+0000 mon.a (mon.0) 1959 : cluster 3 Health check update: Degraded data redundancy: 20/105 objects degraded (19.048%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:10.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:09.267019+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:09.269752+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e349: 8 total, 7 up, 8 in 2023-10-09T04:59:10.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:09.273413+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.543 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:10 smithi153 bash[59565]: cluster 2023-10-09T04:59:09.285427+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T04:59:10.544 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap purge testimg 2023-10-09T04:59:10.560 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:10.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:10.700 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:10.803 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:10.814 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:59:10.836 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:10.940 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:10.966 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:10.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:11.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:11.104 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:11.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:11.239 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:11.344 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:11.357 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:11 smithi153 bash[59565]: cluster 2023-10-09T04:59:10.268512+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.357 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:11 smithi153 bash[59565]: cluster 2023-10-09T04:59:10.269324+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.357 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:11 smithi153 bash[59565]: cluster 2023-10-09T04:59:10.272797+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 7 up, 8 in 2023-10-09T04:59:11.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:11 smithi153 bash[59565]: cluster 2023-10-09T04:59:10.956572+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v1520: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 349 KiB/s rd, 45 KiB/s wr, 470 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:11.379 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:11.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[77063]: cluster 2023-10-09T04:59:10.268512+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[77063]: cluster 2023-10-09T04:59:10.269324+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[77063]: cluster 2023-10-09T04:59:10.272797+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 7 up, 8 in 2023-10-09T04:59:11.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[77063]: cluster 2023-10-09T04:59:10.956572+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v1520: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 349 KiB/s rd, 45 KiB/s wr, 470 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:11.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[79733]: cluster 2023-10-09T04:59:10.268512+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[79733]: cluster 2023-10-09T04:59:10.269324+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T04:59:11.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[79733]: cluster 2023-10-09T04:59:10.272797+0000 mon.a (mon.0) 1961 : cluster 0 osdmap e350: 8 total, 7 up, 8 in 2023-10-09T04:59:11.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:11 smithi082 bash[79733]: cluster 2023-10-09T04:59:10.956572+0000 mgr.y (mgr.34107) 1359 : cluster 0 pgmap v1520: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 349 KiB/s rd, 45 KiB/s wr, 470 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:11.483 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:11.512 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:11.616 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:11.650 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:11.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:11.754 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:11.785 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:11.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:11.889 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:11.920 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:11.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:12.024 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:12.051 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:12.154 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:12.182 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:12.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:12.199 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T04:59:12.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:12.305 INFO:tasks.workunit.client.0.smithi082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:59:12.314 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:12.315 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd snap purge testimg_import 2023-10-09T04:59:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:12.418 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:12.451 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:12.555 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:12.566 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: cluster 2023-10-09T04:59:11.285802+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: cluster 2023-10-09T04:59:11.288519+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: cluster 2023-10-09T04:59:11.290448+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e351: 8 total, 7 up, 8 in 2023-10-09T04:59:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: cluster 2023-10-09T04:59:11.783704+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: audit 2023-10-09T04:59:12.195540+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.82:0/1208012270' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.567 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:12 smithi153 bash[59565]: audit 2023-10-09T04:59:12.196030+0000 mon.a (mon.0) 1963 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.586 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:12.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:12.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: cluster 2023-10-09T04:59:11.285802+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: cluster 2023-10-09T04:59:11.288519+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: cluster 2023-10-09T04:59:11.290448+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e351: 8 total, 7 up, 8 in 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: cluster 2023-10-09T04:59:11.783704+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: audit 2023-10-09T04:59:12.195540+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.82:0/1208012270' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[77063]: audit 2023-10-09T04:59:12.196030+0000 mon.a (mon.0) 1963 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: cluster 2023-10-09T04:59:11.285802+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: cluster 2023-10-09T04:59:11.288519+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: cluster 2023-10-09T04:59:11.290448+0000 mon.a (mon.0) 1962 : cluster 0 osdmap e351: 8 total, 7 up, 8 in 2023-10-09T04:59:12.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: cluster 2023-10-09T04:59:11.783704+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T04:59:12.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: audit 2023-10-09T04:59:12.195540+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.82:0/1208012270' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.703 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:12 smithi082 bash[79733]: audit 2023-10-09T04:59:12.196030+0000 mon.a (mon.0) 1963 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:59:12.718 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:12.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:12.821 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:12.847 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:12.950 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:12.982 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:12.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:13.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:13.106 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:13.209 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:13.237 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:13.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:13.341 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:13.368 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:13.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:13.404 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[79733]: cluster 2023-10-09T04:59:12.296359+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[79733]: cluster 2023-10-09T04:59:12.298507+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e352: 8 total, 7 up, 8 in 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[79733]: cluster 2023-10-09T04:59:12.303102+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[79733]: cluster 2023-10-09T04:59:12.349961+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[79733]: cluster 2023-10-09T04:59:12.958218+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v1523: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 229 op/s; 22/111 objects degraded (19.820%) 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[77063]: cluster 2023-10-09T04:59:12.296359+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.405 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[77063]: cluster 2023-10-09T04:59:12.298507+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e352: 8 total, 7 up, 8 in 2023-10-09T04:59:13.406 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[77063]: cluster 2023-10-09T04:59:12.303102+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.406 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[77063]: cluster 2023-10-09T04:59:12.349961+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.406 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:13 smithi082 bash[77063]: cluster 2023-10-09T04:59:12.958218+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v1523: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 229 op/s; 22/111 objects degraded (19.820%) 2023-10-09T04:59:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T04:59:13.472 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:13.505 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:13.609 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:13.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:13 smithi153 bash[59565]: cluster 2023-10-09T04:59:12.296359+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:13 smithi153 bash[59565]: cluster 2023-10-09T04:59:12.298507+0000 mon.a (mon.0) 1964 : cluster 0 osdmap e352: 8 total, 7 up, 8 in 2023-10-09T04:59:13.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:13 smithi153 bash[59565]: cluster 2023-10-09T04:59:12.303102+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:13 smithi153 bash[59565]: cluster 2023-10-09T04:59:12.349961+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T04:59:13.622 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:13 smithi153 bash[59565]: cluster 2023-10-09T04:59:12.958218+0000 mgr.y (mgr.34107) 1360 : cluster 0 pgmap v1523: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 229 op/s; 22/111 objects degraded (19.820%) 2023-10-09T04:59:13.640 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:13.744 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:13.777 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:13.881 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:13.892 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:13.913 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:13.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:14.018 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:14.150 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:14.179 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:14.282 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:14.313 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:14.326 INFO:tasks.workunit.client.0.smithi082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:59:14.337 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg 2023-10-09T04:59:14.417 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:14.442 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:14.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:14.487 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:14.493 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd remove testimg_import 2023-10-09T04:59:14.545 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:14.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[77063]: cluster 2023-10-09T04:59:13.307557+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[77063]: cluster 2023-10-09T04:59:13.314958+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e353: 8 total, 7 up, 8 in 2023-10-09T04:59:14.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[77063]: cluster 2023-10-09T04:59:13.332545+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[79733]: cluster 2023-10-09T04:59:13.307557+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[79733]: cluster 2023-10-09T04:59:13.314958+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e353: 8 total, 7 up, 8 in 2023-10-09T04:59:14.557 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:14 smithi082 bash[79733]: cluster 2023-10-09T04:59:13.332545+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.571 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:14.657 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:14.663 INFO:tasks.workunit.client.0.smithi082.stderr:+ tiered=0 2023-10-09T04:59:14.663 INFO:tasks.workunit.client.0.smithi082.stderr:+ ceph osd dump 2023-10-09T04:59:14.663 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep ^pool+ 2023-10-09T04:59:14.663 INFO:tasks.workunit.client.0.smithi082.stderr:grep 'rbd' 2023-10-09T04:59:14.663 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep tier 2023-10-09T04:59:14.675 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:14 smithi153 bash[59565]: cluster 2023-10-09T04:59:13.307557+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:14 smithi153 bash[59565]: cluster 2023-10-09T04:59:13.314958+0000 mon.a (mon.0) 1965 : cluster 0 osdmap e353: 8 total, 7 up, 8 in 2023-10-09T04:59:14.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:14 smithi153 bash[59565]: cluster 2023-10-09T04:59:13.332545+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T04:59:14.704 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:14.807 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.835 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:14.905 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:14.939 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.963 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:14.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:15.066 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:15.090 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:15.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:15.193 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:15.220 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:15.323 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:15.354 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:15.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:15.464 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: audit 2023-10-09T04:59:14.310219+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:15.464 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.312917+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.315619+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e354: 8 total, 7 up, 8 in 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.316280+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.320104+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.386403+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:14.958925+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v1526: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[77063]: cluster 2023-10-09T04:59:15.206091+0000 mon.a (mon.0) 1968 : cluster 3 Health check update: Degraded data redundancy: 22/111 objects degraded (19.820%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: audit 2023-10-09T04:59:14.310219+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.312917+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.465 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.315619+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e354: 8 total, 7 up, 8 in 2023-10-09T04:59:15.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.316280+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.320104+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.386403+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:14.958925+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v1526: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:15.466 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:15 smithi082 bash[79733]: cluster 2023-10-09T04:59:15.206091+0000 mon.a (mon.0) 1968 : cluster 3 Health check update: Degraded data redundancy: 22/111 objects degraded (19.820%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:15.476 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:15.538 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_244924/sparse1 2023-10-09T04:59:15.542 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:59:15.542 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:59:15.542 INFO:tasks.workunit.client.0.smithi082.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00359769 s, 291 MB/s 2023-10-09T04:59:15.542 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_244924/sparse2 2023-10-09T04:59:15.546 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:59:15.546 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:59:15.546 INFO:tasks.workunit.client.0.smithi082.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00360015 s, 291 MB/s 2023-10-09T04:59:15.547 INFO:tasks.workunit.client.0.smithi082.stderr:+ truncate /tmp/rbd_import_export_244924/sparse2 -s 2M 2023-10-09T04:59:15.547 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:59:15.579 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:15.589 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:59:15.589 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:59:15.593 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:59:15.593 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_244924/sparse1 2023-10-09T04:59:15.597 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:15.665 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:59:15.666 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:15.666 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:15.670 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:59:15.670 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep sparse1 2023-10-09T04:59:15.671 INFO:tasks.workunit.client.0.smithi082.stderr:grep -Ei (2 MiB|2048k) 2023-10-09T04:59:15.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: audit 2023-10-09T04:59:14.310219+0000 mon.a (mon.0) 1966 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.312917+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.315619+0000 mon.a (mon.0) 1967 : cluster 0 osdmap e354: 8 total, 7 up, 8 in 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.316280+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.320104+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.386403+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:14.958925+0000 mgr.y (mgr.34107) 1361 : cluster 0 pgmap v1526: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 54 KiB/s rd, 5.5 KiB/s wr, 77 op/s; 21/108 objects degraded (19.444%) 2023-10-09T04:59:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:15 smithi153 bash[59565]: cluster 2023-10-09T04:59:15.206091+0000 mon.a (mon.0) 1968 : cluster 3 Health check update: Degraded data redundancy: 22/111 objects degraded (19.820%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:15.701 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:15.715 INFO:tasks.workunit.client.0.smithi082.stdout:sparse1 2 MiB 2 2023-10-09T04:59:15.716 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse1 2023-10-09T04:59:15.716 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:59:15.716 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:59:15.717 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:59:15.717 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:59:15.721 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:15.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:15.757 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.b531c5311c07 2023-10-09T04:59:15.758 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.b531c5311c07 2023-10-09T04:59:15.758 INFO:tasks.workunit.client.0.smithi082.stderr:+ + sed -e s/rbd_data.b531c5311c07\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:15.758 INFO:tasks.workunit.client.0.smithi082.stderr:get_image_data_pool sparse1 2023-10-09T04:59:15.758 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:59:15.759 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:59:15.759 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:59:15.759 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:59:15.759 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:15.801 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:15.801 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:15.801 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:15.801 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:15.801 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:15.824 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:15.840 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:59:15.841 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:59:15.841 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:59:15.841 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:15.855 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:15.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:15.958 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:15.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:15.961 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:15.965 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:15.965 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:15.967 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse1 --format %b 2023-10-09T04:59:15.968 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:59:15.969 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse1.out --format %b 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:15.970 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:59:15.981 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:15.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:16.086 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:16.106 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:59:16.111 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:59:16.115 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:16.164 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 0% complete...failed. 2023-10-09T04:59:16.164 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:59:16.168 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:59:16.168 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_244924/sparse2 2023-10-09T04:59:16.218 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:16.244 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:16.245 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:59:16.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:16.247 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:16.247 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:16.250 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:59:16.250 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep sparse2 2023-10-09T04:59:16.250 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T04:59:16.294 INFO:tasks.workunit.client.0.smithi082.stdout:sparse2 2 MiB 2 2023-10-09T04:59:16.295 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse2 2023-10-09T04:59:16.296 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:59:16.296 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:59:16.296 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:59:16.296 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:59:16.345 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.b564b7b22060 2023-10-09T04:59:16.345 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.b564b7b22060 2023-10-09T04:59:16.346 INFO:tasks.workunit.client.0.smithi082.stderr:+ + + get_image_data_pool sparse2 2023-10-09T04:59:16.346 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:59:16.346 INFO:tasks.workunit.client.0.smithi082.stderr:sedsort -e -u 2023-10-09T04:59:16.346 INFO:tasks.workunit.client.0.smithi082.stderr: s/rbd_data.b564b7b22060\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:16.347 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:59:16.347 INFO:tasks.workunit.client.0.smithi082.stderr:+ + awk -F: { print $NF } 2023-10-09T04:59:16.347 INFO:tasks.workunit.client.0.smithi082.stderr:grep data_pool: 2023-10-09T04:59:16.347 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:16.370 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:16.391 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:16.391 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:16.391 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:16.391 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:16.391 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:16 smithi153 bash[59565]: audit 2023-10-09T04:59:15.394385+0000 mon.a (mon.0) 1969 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:16 smithi153 bash[59565]: audit 2023-10-09T04:59:15.420615+0000 mon.a (mon.0) 1970 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:16 smithi153 bash[59565]: audit 2023-10-09T04:59:15.523794+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.429 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:59:16.429 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2023-10-09T04:59:16.429 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T04:59:16.430 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:16.470 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:16.473 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:16.474 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:16.474 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse2 --format %b 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse2.out --format %b 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:16.479 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:59:16.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[77063]: audit 2023-10-09T04:59:15.394385+0000 mon.a (mon.0) 1969 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:16.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[77063]: audit 2023-10-09T04:59:15.420615+0000 mon.a (mon.0) 1970 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[77063]: audit 2023-10-09T04:59:15.523794+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[79733]: audit 2023-10-09T04:59:15.394385+0000 mon.a (mon.0) 1969 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[79733]: audit 2023-10-09T04:59:15.420615+0000 mon.a (mon.0) 1970 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:16 smithi082 bash[79733]: audit 2023-10-09T04:59:15.523794+0000 mon.a (mon.0) 1971 : audit 0 from='client.? 172.21.15.82:0/2397259063' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:59:16.498 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:16.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:16.594 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:59:16.598 INFO:tasks.workunit.client.0.smithi082.stderr:+ truncate /tmp/rbd_import_export_244924/sparse1 -s 10M 2023-10-09T04:59:16.599 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T04:59:16.602 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:16.637 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:16.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:16.728 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:16.729 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:16.729 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:16.734 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbdgrep ls sparse1 2023-10-09T04:59:16.734 INFO:tasks.workunit.client.0.smithi082.stderr: -l 2023-10-09T04:59:16.734 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T04:59:16.742 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:16.764 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:16.790 INFO:tasks.workunit.client.0.smithi082.stdout:sparse1 10 MiB 2 2023-10-09T04:59:16.791 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse1 2023-10-09T04:59:16.791 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:59:16.791 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:59:16.791 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:59:16.792 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:59:16.840 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.8edd335d8504 2023-10-09T04:59:16.842 INFO:tasks.workunit.client.0.smithi082.stderr:+ + get_image_data_pool sparse1 2023-10-09T04:59:16.842 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse1 2023-10-09T04:59:16.842 INFO:tasks.workunit.client.0.smithi082.stderr:grep rbd_data.8edd335d8504 2023-10-09T04:59:16.842 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.8edd335d8504\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:16.842 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:59:16.843 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse1 2023-10-09T04:59:16.843 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep data_pool: 2023-10-09T04:59:16.843 INFO:tasks.workunit.client.0.smithi082.stderr:awk -F: { print $NF } 2023-10-09T04:59:16.867 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:16.891 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:16.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:16.894 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:16.895 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:16.895 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:16.895 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:16.895 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:16.933 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:59:16.934 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:59:16.934 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:59:16.934 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:16.982 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:16.986 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:16.986 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/sparse1 /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:16.995 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:17.000 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse1 --format %b 2023-10-09T04:59:17.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=2048 2023-10-09T04:59:17.001 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse1.out --format %b 2023-10-09T04:59:17.002 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=2048 2023-10-09T04:59:17.002 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:17.002 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:17.002 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:59:17.002 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse1.out 2023-10-09T04:59:17.003 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse1 2023-10-09T04:59:17.019 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:17.122 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:17.148 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:59:17.149 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:17.152 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_244924/sparse2 oflag=append conv=notrunc 2023-10-09T04:59:17.159 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:59:17.159 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:59:17.159 INFO:tasks.workunit.client.0.smithi082.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00657272 s, 319 MB/s 2023-10-09T04:59:17.159 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T04:59:17.252 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:17.273 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:17.274 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:17.274 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:17.276 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:17.277 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd ls -l 2023-10-09T04:59:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:17.279 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep sparse2 2023-10-09T04:59:17.279 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T04:59:17.320 INFO:tasks.workunit.client.0.smithi082.stdout:sparse2 4 MiB 2 2023-10-09T04:59:17.321 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse2 2023-10-09T04:59:17.321 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:59:17.321 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:59:17.321 INFO:tasks.workunit.client.0.smithi082.stderr:+ + awk {print $NF;} 2023-10-09T04:59:17.321 INFO:tasks.workunit.client.0.smithi082.stderr:rbd info sparse2 2023-10-09T04:59:17.366 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.8f0aa1afd58b 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.8f0aa1afd58b 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.8f0aa1afd58b\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse2 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse2 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse2 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:59:17.368 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:17.379 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:17.397 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:17.418 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:17.418 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:17.418 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:17.418 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:17.419 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:2 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:3 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2 3 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T04:59:17.457 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:17.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:17.502 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T04:59:17.505 INFO:tasks.workunit.client.0.smithi082.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_244924/sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:17.505 INFO:tasks.workunit.client.0.smithi082.stderr:+ cmp -l /tmp/rbd_import_export_244924/sparse2 /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:17.510 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse2 --format %b 2023-10-09T04:59:17.511 INFO:tasks.workunit.client.0.smithi082.stderr:+ origsize=6144 2023-10-09T04:59:17.511 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse2.out --format %b 2023-10-09T04:59:17.511 INFO:tasks.workunit.client.0.smithi082.stderr:+ exportsize=6144 2023-10-09T04:59:17.512 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:17.512 INFO:tasks.workunit.client.0.smithi082.stderr:+ difference=0 2023-10-09T04:59:17.512 INFO:tasks.workunit.client.0.smithi082.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:59:17.512 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse2.out 2023-10-09T04:59:17.512 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse2 2023-10-09T04:59:17.531 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:17.561 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:17 smithi082 bash[79733]: cluster 2023-10-09T04:59:16.959713+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v1527: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 KiB/s rd, 8.5 KiB/s wr, 139 op/s; 19/102 objects degraded (18.627%) 2023-10-09T04:59:17.561 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:17 smithi082 bash[77063]: cluster 2023-10-09T04:59:16.959713+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v1527: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 KiB/s rd, 8.5 KiB/s wr, 139 op/s; 19/102 objects degraded (18.627%) 2023-10-09T04:59:17.635 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:17.648 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T04:59:17.651 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T04:59:17.651 INFO:tasks.workunit.client.0.smithi082.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T04:59:17.652 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_244924/sparse1 sparse 2023-10-09T04:59:17.656 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:17.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:17 smithi153 bash[59565]: cluster 2023-10-09T04:59:16.959713+0000 mgr.y (mgr.34107) 1362 : cluster 0 pgmap v1527: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 4.8 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 KiB/s rd, 8.5 KiB/s wr, 139 op/s; 19/102 objects degraded (18.627%) 2023-10-09T04:59:17.735 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T04:59:17.737 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:17.737 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:59:17.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:59:17.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:17.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:17.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep block_name_prefix 2023-10-09T04:59:17.741 INFO:tasks.workunit.client.0.smithi082.stderr:awk {print $NF;} 2023-10-09T04:59:17.759 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:17.783 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.8f3741ad5de 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep+ rbd_data.8f3741ad5de 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:sort -u 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:+ + rbd info sparse 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:sed+ grep data_pool: 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:17.784 INFO:tasks.workunit.client.0.smithi082.stderr: -e s/rbd_data.8f3741ad5de\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:17.791 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:17.820 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:17.821 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:17.821 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:17.821 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:17.821 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:17.853 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=1 2023-10-09T04:59:17.854 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 1 2023-10-09T04:59:17.854 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T04:59:17.854 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:59:17.895 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:17.924 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:17.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:18.022 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:59:18.025 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T04:59:18.025 INFO:tasks.workunit.client.0.smithi082.stdout:zeros import through stdin to sparse image 2023-10-09T04:59:18.026 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T04:59:18.026 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd import --new-format - sparse 2023-10-09T04:59:18.028 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:18.047 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:18.076 INFO:tasks.workunit.client.0.smithi082.stderr:4+0 records in 2023-10-09T04:59:18.076 INFO:tasks.workunit.client.0.smithi082.stderr:4+0 records out 2023-10-09T04:59:18.077 INFO:tasks.workunit.client.0.smithi082.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0506386 s, 82.8 MB/s 2023-10-09T04:59:18.138 INFO:tasks.workunit.client.0.smithi082.stderr: Importing image: 100% complete...done. 2023-10-09T04:59:18.138 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:18.141 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:59:18.141 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:18.142 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.142 INFO:tasks.workunit.client.0.smithi082.stderr:+ + grep block_name_prefixawk 2023-10-09T04:59:18.142 INFO:tasks.workunit.client.0.smithi082.stderr: {print $NF;} 2023-10-09T04:59:18.150 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:18.180 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:18.181 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.b60c60d499de 2023-10-09T04:59:18.181 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.b60c60d499de 2023-10-09T04:59:18.181 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.b60c60d499de\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:18.181 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:59:18.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:18.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:59:18.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:59:18.182 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:18.220 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:18.220 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:18.220 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:18.220 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:18.220 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:18.253 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects= 2023-10-09T04:59:18.253 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 2023-10-09T04:59:18.253 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T04:59:18.253 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:59:18.283 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:18.304 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:18.407 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [7] live_osds: [0, 1, 3, 4, 6, 5, 2] 2023-10-09T04:59:18.407 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:59:18.408 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T04:59:18.408 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:59:18.408 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:59:18.408 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T04:59:18.409 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:18.430 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:18.460 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:18.464 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo zeros export to sparse file 2023-10-09T04:59:18.464 INFO:tasks.workunit.client.0.smithi082.stdout:zeros export to sparse file 2023-10-09T04:59:18.464 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T04:59:18.513 INFO:tasks.workunit.client.0.smithi082.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:59:18.517 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.518 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep+ block_name_prefix 2023-10-09T04:59:18.518 INFO:tasks.workunit.client.0.smithi082.stderr:awk {print $NF;} 2023-10-09T04:59:18.533 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:18.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.8f7912ebb78d 2023-10-09T04:59:18.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T04:59:18.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:59:18.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:18.561 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:59:18.562 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:18.562 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:18.602 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:18.602 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:18.602 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:18.602 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:18.602 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados -p rbd put rbd_data.8f7912ebb78d.000000000000 - 2023-10-09T04:59:18.632 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records in 2023-10-09T04:59:18.632 INFO:tasks.workunit.client.0.smithi082.stderr:1+0 records out 2023-10-09T04:59:18.632 INFO:tasks.workunit.client.0.smithi082.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0715549 s, 58.6 MB/s 2023-10-09T04:59:18.657 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects sparse 2023-10-09T04:59:18.657 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:18.657 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.657 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk {print $NF;} 2023-10-09T04:59:18.658 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep block_name_prefix 2023-10-09T04:59:18.665 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:18.689 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:18.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:18.698 INFO:tasks.workunit.client.0.smithi082.stderr:+ prefix=rbd_data.8f7912ebb78d 2023-10-09T04:59:18.699 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep rbd_data.8f7912ebb78d 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ get_image_data_pool sparse 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ image=sparse 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ sed -e s/rbd_data.8f7912ebb78d\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ sort -u 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ grep data_pool: 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ awk -F: { print $NF } 2023-10-09T04:59:18.700 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd info sparse 2023-10-09T04:59:18.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool= 2023-10-09T04:59:18.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ -z ] 2023-10-09T04:59:18.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ data_pool=rbd 2023-10-09T04:59:18.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo rbd 2023-10-09T04:59:18.741 INFO:tasks.workunit.client.0.smithi082.stderr:+ rados ls -p rbd 2023-10-09T04:59:18.780 INFO:tasks.workunit.client.0.smithi082.stderr:+ objects=0 2023-10-09T04:59:18.780 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo 0 2023-10-09T04:59:18.780 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T04:59:18.780 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse 2023-10-09T04:59:18.781 INFO:tasks.workunit.client.0.smithi082.stderr:rm: cannot remove '/tmp/rbd_import_export_244924/sparse': No such file or directory 2023-10-09T04:59:18.781 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:59:18.781 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd export sparse /tmp/rbd_import_export_244924/sparse 2023-10-09T04:59:18.793 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:18.824 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:18.825 INFO:tasks.workunit.client.0.smithi082.stderr: Exporting image: 100% complete...done. 2023-10-09T04:59:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:18.830 INFO:tasks.workunit.client.0.smithi082.stderr:+ stat /tmp/rbd_import_export_244924/sparse --format=%b 2023-10-09T04:59:18.831 INFO:tasks.workunit.client.0.smithi082.stderr:+ [ 0 = 0 ] 2023-10-09T04:59:18.831 INFO:tasks.workunit.client.0.smithi082.stderr:+ rbd rm sparse 2023-10-09T04:59:18.929 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:18.954 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:18.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:18.959 INFO:tasks.workunit.client.0.smithi082.stderr: Removing image: 100% complete...done. 2023-10-09T04:59:18.964 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm /tmp/rbd_import_export_244924/sparse /tmp/rbd_import_export_244924/sparse1 /tmp/rbd_import_export_244924/sparse2 /tmp/rbd_import_export_244924/sparse3 2023-10-09T04:59:18.968 INFO:tasks.workunit.client.0.smithi082.stderr:rm: cannot remove '/tmp/rbd_import_export_244924/sparse3': No such file or directory 2023-10-09T04:59:18.968 INFO:tasks.workunit.client.0.smithi082.stderr:+ true 2023-10-09T04:59:18.968 INFO:tasks.workunit.client.0.smithi082.stderr:+ echo OK 2023-10-09T04:59:18.968 INFO:tasks.workunit.client.0.smithi082.stdout:OK 2023-10-09T04:59:18.969 INFO:tasks.workunit.client.0.smithi082.stderr:+ rm -rf /tmp/rbd_import_export_244924 2023-10-09T04:59:18.969 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T04:59:18.970 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T04:59:19.030 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T04:59:19.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:59:19.057 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:19.103 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:19.207 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:19.238 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:19.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:19.345 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:19.375 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:19.478 DEBUG:teuthology.parallel:result is None 2023-10-09T04:59:19.479 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:59:19.479 DEBUG:teuthology.orchestra.run.smithi153:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:19.488 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:59:19.488 DEBUG:teuthology.orchestra.run.smithi082:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:59:19.513 INFO:teuthology.orchestra.run.smithi153.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi153 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:19.535 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:59:19.535 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:59:19.535 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T04:59:19.535 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:59:19.536 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T04:59:19.536 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:59:19.536 DEBUG:teuthology.orchestra.run.smithi082:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:59:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.583 INFO:teuthology.orchestra.run.smithi082.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:59:19.584 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:59:19.617 DEBUG:teuthology.orchestra.run.smithi082:> sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:19.631 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:59:19.631 DEBUG:teuthology.orchestra.run.smithi082:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:59:19.652 INFO:teuthology.orchestra.run.smithi082.stderr:Error response from daemon: Cannot kill container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:59:19.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi082 with status 1: 'sudo docker kill -s 1 ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:19.658 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T04:59:19.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:19] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.33.4" 2023-10-09T04:59:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:20 smithi153 bash[59565]: cluster 2023-10-09T04:59:18.960776+0000 mgr.y (mgr.34107) 1363 : cluster 0 pgmap v1528: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 989 KiB/s rd, 935 KiB/s wr, 298 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:20.474 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:20 smithi082 bash[77063]: cluster 2023-10-09T04:59:18.960776+0000 mgr.y (mgr.34107) 1363 : cluster 0 pgmap v1528: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 989 KiB/s rd, 935 KiB/s wr, 298 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:20.474 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:20 smithi082 bash[79733]: cluster 2023-10-09T04:59:18.960776+0000 mgr.y (mgr.34107) 1363 : cluster 0 pgmap v1528: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 989 KiB/s rd, 935 KiB/s wr, 298 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:21.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:21 smithi153 bash[59565]: cluster 2023-10-09T04:59:20.207548+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 21/96 objects degraded (21.875%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:21 smithi082 bash[77063]: cluster 2023-10-09T04:59:20.207548+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 21/96 objects degraded (21.875%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:21 smithi082 bash[79733]: cluster 2023-10-09T04:59:20.207548+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 21/96 objects degraded (21.875%), 12 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:22.293 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[79733]: cluster 2023-10-09T04:59:20.961716+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v1529: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 987 KiB/s rd, 911 KiB/s wr, 296 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[79733]: audit 2023-10-09T04:59:21.633898+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[79733]: audit 2023-10-09T04:59:21.640200+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[77063]: cluster 2023-10-09T04:59:20.961716+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v1529: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 987 KiB/s rd, 911 KiB/s wr, 296 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[77063]: audit 2023-10-09T04:59:21.633898+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:22.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:22 smithi082 bash[77063]: audit 2023-10-09T04:59:21.640200+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:22.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:22 smithi153 bash[59565]: cluster 2023-10-09T04:59:20.961716+0000 mgr.y (mgr.34107) 1364 : cluster 0 pgmap v1529: 73 pgs: 18 active+undersized, 12 active+undersized+degraded, 43 active+clean; 5.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 987 KiB/s rd, 911 KiB/s wr, 296 op/s; 21/96 objects degraded (21.875%) 2023-10-09T04:59:22.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:22 smithi153 bash[59565]: audit 2023-10-09T04:59:21.633898+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:22.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:22 smithi153 bash[59565]: audit 2023-10-09T04:59:21.640200+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.667 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: audit 2023-10-09T04:59:22.453374+0000 mon.a (mon.0) 1975 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.668 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: audit 2023-10-09T04:59:22.459163+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.668 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: audit 2023-10-09T04:59:22.927389+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:23.668 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: audit 2023-10-09T04:59:22.928646+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:23.668 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: audit 2023-10-09T04:59:22.937873+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.668 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:23 smithi153 bash[59565]: cluster 2023-10-09T04:59:22.962351+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v1530: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 863 KiB/s rd, 1.2 MiB/s wr, 302 op/s; 19/90 objects degraded (21.111%) 2023-10-09T04:59:23.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: audit 2023-10-09T04:59:22.453374+0000 mon.a (mon.0) 1975 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.718 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: audit 2023-10-09T04:59:22.459163+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: audit 2023-10-09T04:59:22.927389+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: audit 2023-10-09T04:59:22.928646+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: audit 2023-10-09T04:59:22.937873+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[77063]: cluster 2023-10-09T04:59:22.962351+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v1530: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 863 KiB/s rd, 1.2 MiB/s wr, 302 op/s; 19/90 objects degraded (21.111%) 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: audit 2023-10-09T04:59:22.453374+0000 mon.a (mon.0) 1975 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: audit 2023-10-09T04:59:22.459163+0000 mon.a (mon.0) 1976 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: audit 2023-10-09T04:59:22.927389+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:59:23.719 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: audit 2023-10-09T04:59:22.928646+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:59:23.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: audit 2023-10-09T04:59:22.937873+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T04:59:23.720 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:23 smithi082 bash[79733]: cluster 2023-10-09T04:59:22.962351+0000 mgr.y (mgr.34107) 1365 : cluster 0 pgmap v1530: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 863 KiB/s rd, 1.2 MiB/s wr, 302 op/s; 19/90 objects degraded (21.111%) 2023-10-09T04:59:24.143 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2023-10-09T04:59:24.143 INFO:teuthology.orchestra.run.smithi082.stdout: "ops": [], 2023-10-09T04:59:24.143 INFO:teuthology.orchestra.run.smithi082.stdout: "complaint_time": 30, 2023-10-09T04:59:24.144 INFO:teuthology.orchestra.run.smithi082.stdout: "num_blocked_ops": 0 2023-10-09T04:59:24.144 INFO:teuthology.orchestra.run.smithi082.stdout:} 2023-10-09T04:59:25.482 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:59:25.482 INFO:teuthology.orchestra.run.smithi153.stdout: "success": "" 2023-10-09T04:59:25.482 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:59:25.819 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:59:26.121 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:26 smithi082 bash[77063]: cluster 2023-10-09T04:59:24.963608+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v1531: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 273 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:26.121 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:26 smithi082 bash[77063]: cluster 2023-10-09T04:59:25.209166+0000 mon.a (mon.0) 1980 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.121 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:26 smithi082 bash[79733]: cluster 2023-10-09T04:59:24.963608+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v1531: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 273 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:26.121 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:26 smithi082 bash[79733]: cluster 2023-10-09T04:59:25.209166+0000 mon.a (mon.0) 1980 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.340 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0: {} 2023-10-09T04:59:26.368 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1: {} 2023-10-09T04:59:26.394 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2: {} 2023-10-09T04:59:26.418 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3: {} 2023-10-09T04:59:26.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:26 smithi153 bash[59565]: cluster 2023-10-09T04:59:24.963608+0000 mgr.y (mgr.34107) 1366 : cluster 0 pgmap v1531: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 782 KiB/s rd, 1.0 MiB/s wr, 273 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:26.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:26 smithi153 bash[59565]: cluster 2023-10-09T04:59:25.209166+0000 mon.a (mon.0) 1980 : cluster 3 Health check update: Degraded data redundancy: 18/87 objects degraded (20.690%), 11 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.442 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4: {} 2023-10-09T04:59:26.468 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5: {} 2023-10-09T04:59:26.493 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6: {} 2023-10-09T04:59:26.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.494 INFO:teuthology.orchestra.run.smithi082.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.495 INFO:teuthology.orchestra.run.smithi082.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.495 INFO:teuthology.orchestra.run.smithi082.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.495 INFO:teuthology.orchestra.run.smithi082.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:59:26.495 INFO:teuthology.orchestra.run.smithi082.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T04:59:27.035 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:59:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:27 smithi153 bash[59565]: audit 2023-10-09T04:59:26.311976+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.82:0/203827242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:27.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:27 smithi153 bash[59565]: audit 2023-10-09T04:59:26.562622+0000 mon.a (mon.0) 1981 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:27.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:27 smithi082 bash[77063]: audit 2023-10-09T04:59:26.311976+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.82:0/203827242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:27.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:27 smithi082 bash[77063]: audit 2023-10-09T04:59:26.562622+0000 mon.a (mon.0) 1981 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:27.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:27 smithi082 bash[79733]: audit 2023-10-09T04:59:26.311976+0000 mon.c (mon.2) 147 : audit 0 from='client.? 172.21.15.82:0/203827242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:27.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:27 smithi082 bash[79733]: audit 2023-10-09T04:59:26.562622+0000 mon.a (mon.0) 1981 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:59:27.670 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:59:28.290 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[79733]: cluster 2023-10-09T04:59:26.965159+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v1532: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 694 KiB/s rd, 949 KiB/s wr, 242 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[79733]: cluster 2023-10-09T04:59:27.018796+0000 mon.a (mon.0) 1982 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[79733]: cluster 2023-10-09T04:59:27.027500+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[79733]: audit 2023-10-09T04:59:27.031493+0000 mon.a (mon.0) 1983 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[79733]: cluster 2023-10-09T04:59:27.031581+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e355: 8 total, 7 up, 8 in 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[77063]: cluster 2023-10-09T04:59:26.965159+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v1532: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 694 KiB/s rd, 949 KiB/s wr, 242 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[77063]: cluster 2023-10-09T04:59:27.018796+0000 mon.a (mon.0) 1982 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:28.291 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[77063]: cluster 2023-10-09T04:59:27.027500+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T04:59:28.292 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[77063]: audit 2023-10-09T04:59:27.031493+0000 mon.a (mon.0) 1983 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:28.292 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:28 smithi082 bash[77063]: cluster 2023-10-09T04:59:27.031581+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e355: 8 total, 7 up, 8 in 2023-10-09T04:59:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:28 smithi153 bash[59565]: cluster 2023-10-09T04:59:26.965159+0000 mgr.y (mgr.34107) 1367 : cluster 0 pgmap v1532: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 694 KiB/s rd, 949 KiB/s wr, 242 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:28 smithi153 bash[59565]: cluster 2023-10-09T04:59:27.018796+0000 mon.a (mon.0) 1982 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:28 smithi153 bash[59565]: cluster 2023-10-09T04:59:27.027500+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T04:59:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:28 smithi153 bash[59565]: audit 2023-10-09T04:59:27.031493+0000 mon.a (mon.0) 1983 : audit 1 from='client.? 172.21.15.82:0/4100486075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:28.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:28 smithi153 bash[59565]: cluster 2023-10-09T04:59:27.031581+0000 mon.a (mon.0) 1984 : cluster 0 osdmap e355: 8 total, 7 up, 8 in 2023-10-09T04:59:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:29] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T04:59:30.293 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:59:30.304 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:30 smithi082 bash[77063]: cluster 2023-10-09T04:59:28.966651+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v1534: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 195 KiB/s rd, 518 KiB/s wr, 122 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:30.305 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:30 smithi082 bash[77063]: audit 2023-10-09T04:59:29.310004+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:30.305 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:30 smithi082 bash[79733]: cluster 2023-10-09T04:59:28.966651+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v1534: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 195 KiB/s rd, 518 KiB/s wr, 122 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:30.305 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:30 smithi082 bash[79733]: audit 2023-10-09T04:59:29.310004+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:30 smithi153 bash[59565]: cluster 2023-10-09T04:59:28.966651+0000 mgr.y (mgr.34107) 1368 : cluster 0 pgmap v1534: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 195 KiB/s rd, 518 KiB/s wr, 122 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:30 smithi153 bash[59565]: audit 2023-10-09T04:59:29.310004+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:31.238 INFO:tasks.thrashosds.thrasher:reviving osd 2023-10-09T04:59:31.238 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-10-09T04:59:31.238 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-10-09T04:59:31.238 DEBUG:teuthology.orchestra.run.smithi153:osd.7> sudo journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T04:59:31.241 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl start ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T04:59:31.306 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:59:31.350 INFO:journalctl@ceph.osd.7.smithi153.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T04:59:31.813 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:32.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:32 smithi082 bash[77063]: cluster 2023-10-09T04:59:30.968169+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v1535: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 44 KiB/s rd, 410 KiB/s wr, 55 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:32.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:32 smithi082 bash[79733]: cluster 2023-10-09T04:59:30.968169+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v1535: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 44 KiB/s rd, 410 KiB/s wr, 55 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:32.510 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:32 smithi153 bash[59565]: cluster 2023-10-09T04:59:30.968169+0000 mgr.y (mgr.34107) 1369 : cluster 0 pgmap v1535: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 44 KiB/s rd, 410 KiB/s wr, 55 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-10-09T04:59:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:33 smithi153 bash[152742]: --> ceph-volume raw activate successful for osd ID: 7 2023-10-09T04:59:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:34 smithi153 bash[59565]: cluster 2023-10-09T04:59:32.969116+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v1536: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 306 B/s rd, 0 B/s wr, 0 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:34 smithi082 bash[77063]: cluster 2023-10-09T04:59:32.969116+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v1536: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 306 B/s rd, 0 B/s wr, 0 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:34 smithi082 bash[79733]: cluster 2023-10-09T04:59:32.969116+0000 mgr.y (mgr.34107) 1370 : cluster 0 pgmap v1536: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 306 B/s rd, 0 B/s wr, 0 op/s; 18/87 objects degraded (20.690%) 2023-10-09T04:59:34.859 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:35.175 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:35 smithi153 bash[153020]: debug 2023-10-09T04:59:35.041+0000 7f69f40d4700 1 -- 172.21.15.153:0/2639048713 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fad34184e0 con 0x55fad348ac00 2023-10-09T04:59:35.925 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:35 smithi153 bash[153020]: debug 2023-10-09T04:59:35.585+0000 7f69fbb4f200 -1 Falling back to public interface 2023-10-09T04:59:36.285 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:36.286 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:36.331 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:36 smithi082 bash[79733]: cluster 2023-10-09T04:59:34.970430+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v1537: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:36.331 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:36 smithi082 bash[77063]: cluster 2023-10-09T04:59:34.970430+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v1537: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:36 smithi153 bash[59565]: cluster 2023-10-09T04:59:34.970430+0000 mgr.y (mgr.34107) 1371 : cluster 0 pgmap v1537: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:37.537 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:37.835 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:59:38.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:38 smithi082 bash[79733]: cluster 2023-10-09T04:59:36.971840+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v1538: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:38.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:38 smithi082 bash[77063]: cluster 2023-10-09T04:59:36.971840+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v1538: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:38.658 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:38 smithi153 bash[59565]: cluster 2023-10-09T04:59:36.971840+0000 mgr.y (mgr.34107) 1372 : cluster 0 pgmap v1538: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:39] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T04:59:40.248 INFO:teuthology.orchestra.run.smithi082.stderr:noscrub is unset 2023-10-09T04:59:40.541 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:40 smithi153 bash[59565]: cluster 2023-10-09T04:59:38.972582+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v1539: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:40.542 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:40 smithi153 bash[59565]: audit 2023-10-09T04:59:39.243924+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:40 smithi082 bash[79733]: cluster 2023-10-09T04:59:38.972582+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v1539: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:40 smithi082 bash[79733]: audit 2023-10-09T04:59:39.243924+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:40 smithi082 bash[77063]: cluster 2023-10-09T04:59:38.972582+0000 mgr.y (mgr.34107) 1373 : cluster 0 pgmap v1539: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:40 smithi082 bash[77063]: audit 2023-10-09T04:59:39.243924+0000 mon.a (mon.0) 1986 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:40.745 INFO:teuthology.orchestra.run.smithi153.stderr:no valid command found; 10 closest matches: 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:0 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:1 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:2 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:abort 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:assert 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs debug_inject_read_zeros 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs files list 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:bluefs stats 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:bluestore bluefs device info [] 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:config diff 2023-10-09T04:59:40.746 INFO:teuthology.orchestra.run.smithi153.stderr:admin_socket: invalid command 2023-10-09T04:59:41.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:59:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[77063]: cluster 2023-10-09T04:59:40.241410+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[77063]: audit 2023-10-09T04:59:40.243457+0000 mon.a (mon.0) 1987 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[77063]: cluster 2023-10-09T04:59:40.243564+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 7 up, 8 in 2023-10-09T04:59:41.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[79733]: cluster 2023-10-09T04:59:40.241410+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[79733]: audit 2023-10-09T04:59:40.243457+0000 mon.a (mon.0) 1987 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:41 smithi082 bash[79733]: cluster 2023-10-09T04:59:40.243564+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 7 up, 8 in 2023-10-09T04:59:41.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:41 smithi153 bash[59565]: cluster 2023-10-09T04:59:40.241410+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:41.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:41 smithi153 bash[59565]: audit 2023-10-09T04:59:40.243457+0000 mon.a (mon.0) 1987 : audit 1 from='client.? 172.21.15.82:0/723869848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:41.636 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:41 smithi153 bash[59565]: cluster 2023-10-09T04:59:40.243564+0000 mon.a (mon.0) 1988 : cluster 0 osdmap e356: 8 total, 7 up, 8 in 2023-10-09T04:59:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:59:41.637 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-10-09T04:59:42.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:42 smithi153 bash[59565]: cluster 2023-10-09T04:59:40.779283+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:42.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:42 smithi153 bash[59565]: cluster 2023-10-09T04:59:40.973320+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v1541: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:42 smithi082 bash[79733]: cluster 2023-10-09T04:59:40.779283+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:42 smithi082 bash[79733]: cluster 2023-10-09T04:59:40.973320+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v1541: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:42 smithi082 bash[77063]: cluster 2023-10-09T04:59:40.779283+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T04:59:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:42 smithi082 bash[77063]: cluster 2023-10-09T04:59:40.973320+0000 mgr.y (mgr.34107) 1374 : cluster 0 pgmap v1541: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:43.128 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:44.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:44 smithi153 bash[59565]: cluster 2023-10-09T04:59:42.974711+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v1542: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:44 smithi082 bash[77063]: cluster 2023-10-09T04:59:42.974711+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v1542: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:44 smithi082 bash[79733]: cluster 2023-10-09T04:59:42.974711+0000 mgr.y (mgr.34107) 1375 : cluster 0 pgmap v1542: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:45 smithi153 bash[59565]: audit 2023-10-09T04:59:44.310244+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:45 smithi153 bash[59565]: audit 2023-10-09T04:59:45.207716+0000 mon.b (mon.1) 35 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:45 smithi153 bash[59565]: audit 2023-10-09T04:59:45.208511+0000 mon.a (mon.0) 1990 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:45 smithi153 bash[153020]: debug 2023-10-09T04:59:45.201+0000 7f69fbb4f200 -1 osd.7 320 log_to_monitors {default=true} 2023-10-09T04:59:45.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 04:59:45 smithi153 bash[153020]: debug 2023-10-09T04:59:45.309+0000 7f69eb045700 -1 osd.7 320 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:59:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[77063]: audit 2023-10-09T04:59:44.310244+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[77063]: audit 2023-10-09T04:59:45.207716+0000 mon.b (mon.1) 35 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[77063]: audit 2023-10-09T04:59:45.208511+0000 mon.a (mon.0) 1990 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[79733]: audit 2023-10-09T04:59:44.310244+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[79733]: audit 2023-10-09T04:59:45.207716+0000 mon.b (mon.1) 35 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:45 smithi082 bash[79733]: audit 2023-10-09T04:59:45.208511+0000 mon.a (mon.0) 1990 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:59:45.938 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:46.593 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:46.593 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: cluster 2023-10-09T04:59:44.976109+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v1543: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: cluster 2023-10-09T04:59:45.264841+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: audit 2023-10-09T04:59:45.268385+0000 mon.a (mon.0) 1991 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: cluster 2023-10-09T04:59:45.268483+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e357: 8 total, 7 up, 8 in 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: audit 2023-10-09T04:59:45.269945+0000 mon.b (mon.1) 36 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:46.617 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:46 smithi153 bash[59565]: audit 2023-10-09T04:59:45.270911+0000 mon.a (mon.0) 1993 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:46.638 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:59:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: cluster 2023-10-09T04:59:44.976109+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v1543: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: cluster 2023-10-09T04:59:45.264841+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: audit 2023-10-09T04:59:45.268385+0000 mon.a (mon.0) 1991 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:59:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: cluster 2023-10-09T04:59:45.268483+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e357: 8 total, 7 up, 8 in 2023-10-09T04:59:46.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: audit 2023-10-09T04:59:45.269945+0000 mon.b (mon.1) 36 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[79733]: audit 2023-10-09T04:59:45.270911+0000 mon.a (mon.0) 1993 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: cluster 2023-10-09T04:59:44.976109+0000 mgr.y (mgr.34107) 1376 : cluster 0 pgmap v1543: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: cluster 2023-10-09T04:59:45.264841+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: audit 2023-10-09T04:59:45.268385+0000 mon.a (mon.0) 1991 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: cluster 2023-10-09T04:59:45.268483+0000 mon.a (mon.0) 1992 : cluster 0 osdmap e357: 8 total, 7 up, 8 in 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: audit 2023-10-09T04:59:45.269945+0000 mon.b (mon.1) 36 : audit 1 from='osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:46 smithi082 bash[77063]: audit 2023-10-09T04:59:45.270911+0000 mon.a (mon.0) 1993 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2023-10-09T04:59:47.524 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.061091+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.064301+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.273150+0000 mon.a (mon.0) 1994 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.275957+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.276143+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.278296+0000 mon.a (mon.0) 1995 : cluster 1 osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735] boot 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.278380+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: audit 2023-10-09T04:59:46.279487+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.977313+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v1546: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:47.525 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.061091+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.064301+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.273150+0000 mon.a (mon.0) 1994 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.275957+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.276143+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.278296+0000 mon.a (mon.0) 1995 : cluster 1 osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735] boot 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.278380+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: audit 2023-10-09T04:59:46.279487+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:59:47.526 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:47 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.977313+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v1546: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.061091+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.064301+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.273150+0000 mon.a (mon.0) 1994 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.275957+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.276143+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.278296+0000 mon.a (mon.0) 1995 : cluster 1 osd.7 [v2:172.21.15.153:6824/3882514735,v1:172.21.15.153:6825/3882514735] boot 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.278380+0000 mon.a (mon.0) 1996 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-09T04:59:47.614 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: audit 2023-10-09T04:59:46.279487+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:59:47.615 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:47 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.977313+0000 mgr.y (mgr.34107) 1377 : cluster 0 pgmap v1546: 73 pgs: 19 active+undersized, 11 active+undersized+degraded, 43 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 18/87 objects degraded (20.690%) 2023-10-09T04:59:47.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:59:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.198902+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:59:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:46.279347+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.281308+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.281838+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.283412+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.284789+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T04:59:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.285532+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:48 smithi153 bash[59565]: cluster 2023-10-09T04:59:47.292135+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.198902+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:59:48.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:46.279347+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:48.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.281308+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.281838+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.283412+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.284789+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T04:59:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.285532+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[77063]: cluster 2023-10-09T04:59:47.292135+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.198902+0000 osd.7 (osd.7) 1 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T04:59:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:46.279347+0000 osd.7 (osd.7) 2 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T04:59:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.281308+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.281838+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.283412+0000 osd.7 (osd.7) 3 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.284789+0000 mon.a (mon.0) 1998 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-09T04:59:48.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.285532+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:48.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:48 smithi082 bash[79733]: cluster 2023-10-09T04:59:47.292135+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T04:59:49.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.291697+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.295630+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T04:59:49.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.299125+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.461909+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.575 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.461996+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.576 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[79733]: cluster 2023-10-09T04:59:48.978579+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v1549: 73 pgs: 3 remapped+peering, 10 active+undersized, 9 active+undersized+degraded, 51 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/87 objects degraded (18.391%); 0 B/s, 2 objects/s recovering 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.291697+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.295630+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.299125+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.461909+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.461996+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.578 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[77063]: cluster 2023-10-09T04:59:48.978579+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v1549: 73 pgs: 3 remapped+peering, 10 active+undersized, 9 active+undersized+degraded, 51 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/87 objects degraded (18.391%); 0 B/s, 2 objects/s recovering 2023-10-09T04:59:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.291697+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.295630+0000 mon.a (mon.0) 1999 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-09T04:59:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.299125+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.461909+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.461996+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T04:59:49.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:49 smithi153 bash[59565]: cluster 2023-10-09T04:59:48.978579+0000 mgr.y (mgr.34107) 1378 : cluster 0 pgmap v1549: 73 pgs: 3 remapped+peering, 10 active+undersized, 9 active+undersized+degraded, 51 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 16/87 objects degraded (18.391%); 0 B/s, 2 objects/s recovering 2023-10-09T04:59:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:49] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T04:59:50.323 INFO:teuthology.orchestra.run.smithi082.stderr:nodeep-scrub is unset 2023-10-09T04:59:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.290243+0000 mon.a (mon.0) 2000 : cluster 3 Health check update: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.300342+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.301212+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.303322+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.306539+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.307058+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: cluster 2023-10-09T04:59:49.565111+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: audit 2023-10-09T04:59:49.949798+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.82:0/1892494296' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:50 smithi153 bash[59565]: audit 2023-10-09T04:59:49.950461+0000 mon.a (mon.0) 2002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.290243+0000 mon.a (mon.0) 2000 : cluster 3 Health check update: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:50.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.300342+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.301212+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.303322+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.306539+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T04:59:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.307058+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: cluster 2023-10-09T04:59:49.565111+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: audit 2023-10-09T04:59:49.949798+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.82:0/1892494296' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[77063]: audit 2023-10-09T04:59:49.950461+0000 mon.a (mon.0) 2002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.290243+0000 mon.a (mon.0) 2000 : cluster 3 Health check update: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:50.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.300342+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.301212+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.303322+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.306539+0000 mon.a (mon.0) 2001 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.307058+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: cluster 2023-10-09T04:59:49.565111+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: audit 2023-10-09T04:59:49.949798+0000 mon.c (mon.2) 148 : audit 1 from='client.? 172.21.15.82:0/1892494296' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:50 smithi082 bash[79733]: audit 2023-10-09T04:59:49.950461+0000 mon.a (mon.0) 2002 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:51.413 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:51.413 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:51.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.299161+0000 mon.a (mon.0) 2003 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.303185+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: audit 2023-10-09T04:59:50.307451+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.307623+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.308370+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.313780+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.674592+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[77063]: cluster 2023-10-09T04:59:50.979414+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v1552: 73 pgs: 3 peering, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:51.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.299161+0000 mon.a (mon.0) 2003 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.303185+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: audit 2023-10-09T04:59:50.307451+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.307623+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.308370+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.313780+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.674592+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:51 smithi082 bash[79733]: cluster 2023-10-09T04:59:50.979414+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v1552: 73 pgs: 3 peering, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.299161+0000 mon.a (mon.0) 2003 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.303185+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: audit 2023-10-09T04:59:50.307451+0000 mon.a (mon.0) 2004 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.307623+0000 mon.a (mon.0) 2005 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-09T04:59:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.308370+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.313780+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.674592+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T04:59:51.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:51 smithi153 bash[59565]: cluster 2023-10-09T04:59:50.979414+0000 mgr.y (mgr.34107) 1379 : cluster 0 pgmap v1552: 73 pgs: 3 peering, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:52.682 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:52 smithi153 bash[59565]: cluster 2023-10-09T04:59:51.315417+0000 mon.a (mon.0) 2006 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized) 2023-10-09T04:59:52.683 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:52 smithi153 bash[59565]: cluster 2023-10-09T04:59:51.315495+0000 mon.a (mon.0) 2007 : cluster 1 Cluster is now healthy 2023-10-09T04:59:52.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:52 smithi082 bash[79733]: cluster 2023-10-09T04:59:51.315417+0000 mon.a (mon.0) 2006 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized) 2023-10-09T04:59:52.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:52 smithi082 bash[79733]: cluster 2023-10-09T04:59:51.315495+0000 mon.a (mon.0) 2007 : cluster 1 Cluster is now healthy 2023-10-09T04:59:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:52 smithi082 bash[77063]: cluster 2023-10-09T04:59:51.315417+0000 mon.a (mon.0) 2006 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/87 objects degraded (18.391%), 9 pgs degraded, 19 pgs undersized) 2023-10-09T04:59:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:52 smithi082 bash[77063]: cluster 2023-10-09T04:59:51.315495+0000 mon.a (mon.0) 2007 : cluster 1 Cluster is now healthy 2023-10-09T04:59:53.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:53 smithi153 bash[59565]: cluster 2023-10-09T04:59:52.980188+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v1553: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:53.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:53 smithi082 bash[77063]: cluster 2023-10-09T04:59:52.980188+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v1553: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:53.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:53 smithi082 bash[79733]: cluster 2023-10-09T04:59:52.980188+0000 mgr.y (mgr.34107) 1380 : cluster 0 pgmap v1553: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:56.016 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T04:59:56.016 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T04:59:56.016 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T04:59:56.016 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T04:59:56.045 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:56.045 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:56.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:56 smithi153 bash[59565]: cluster 2023-10-09T04:59:54.980984+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v1554: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:56.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:56 smithi082 bash[77063]: cluster 2023-10-09T04:59:54.980984+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v1554: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:56 smithi082 bash[79733]: cluster 2023-10-09T04:59:54.980984+0000 mgr.y (mgr.34107) 1381 : cluster 0 pgmap v1554: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 4 objects/s recovering 2023-10-09T04:59:57.487 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T04:59:57.528 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:59:58.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 04:59:58 smithi082 bash[77063]: cluster 2023-10-09T04:59:56.981903+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v1555: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 4 objects/s recovering 2023-10-09T04:59:58.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 04:59:58 smithi082 bash[79733]: cluster 2023-10-09T04:59:56.981903+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v1555: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 4 objects/s recovering 2023-10-09T04:59:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 04:59:58 smithi153 bash[59565]: cluster 2023-10-09T04:59:56.981903+0000 mgr.y (mgr.34107) 1382 : cluster 0 pgmap v1555: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 512 KiB/s, 4 objects/s recovering 2023-10-09T04:59:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 04:59:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:04:59:59] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.33.4" 2023-10-09T05:00:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:00 smithi153 bash[59565]: cluster 2023-10-09T04:59:58.982759+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 423 KiB/s, 4 objects/s recovering 2023-10-09T05:00:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:00 smithi153 bash[59565]: audit 2023-10-09T04:59:59.317219+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:00 smithi153 bash[59565]: audit 2023-10-09T04:59:59.319199+0000 mon.a (mon.0) 2009 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:00 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.000172+0000 mon.a (mon.0) 2010 : cluster 1 overall HEALTH_OK 2023-10-09T05:00:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:00 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.228844+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:00:00.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[79733]: cluster 2023-10-09T04:59:58.982759+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 423 KiB/s, 4 objects/s recovering 2023-10-09T05:00:00.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[79733]: audit 2023-10-09T04:59:59.317219+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[79733]: audit 2023-10-09T04:59:59.319199+0000 mon.a (mon.0) 2009 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.000172+0000 mon.a (mon.0) 2010 : cluster 1 overall HEALTH_OK 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.228844+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[77063]: cluster 2023-10-09T04:59:58.982759+0000 mgr.y (mgr.34107) 1383 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 423 KiB/s, 4 objects/s recovering 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[77063]: audit 2023-10-09T04:59:59.317219+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[77063]: audit 2023-10-09T04:59:59.319199+0000 mon.a (mon.0) 2009 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:00.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.000172+0000 mon.a (mon.0) 2010 : cluster 1 overall HEALTH_OK 2023-10-09T05:00:00.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:00 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.228844+0000 mon.a (mon.0) 2011 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:00:01.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:01 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.226367+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:01 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.226504+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:01 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.346200+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.226367+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.226504+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.346200+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.226367+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.226504+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:01.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:01 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.346200+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T05:00:02.269 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:02.310 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:02 smithi153 bash[59565]: cluster 2023-10-09T05:00:00.983668+0000 mgr.y (mgr.34107) 1384 : cluster 0 pgmap v1558: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 3 objects/s recovering 2023-10-09T05:00:02.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:02 smithi082 bash[77063]: cluster 2023-10-09T05:00:00.983668+0000 mgr.y (mgr.34107) 1384 : cluster 0 pgmap v1558: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 3 objects/s recovering 2023-10-09T05:00:02.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:02 smithi082 bash[79733]: cluster 2023-10-09T05:00:00.983668+0000 mgr.y (mgr.34107) 1384 : cluster 0 pgmap v1558: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 3 objects/s recovering 2023-10-09T05:00:04.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:04 smithi153 bash[59565]: cluster 2023-10-09T05:00:02.984420+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:04.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:04 smithi082 bash[79733]: cluster 2023-10-09T05:00:02.984420+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:04.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:04 smithi082 bash[77063]: cluster 2023-10-09T05:00:02.984420+0000 mgr.y (mgr.34107) 1385 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:05.604 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:05.604 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:05.761 INFO:teuthology.orchestra.run.smithi082.stdout:mon.a: {} 2023-10-09T05:00:05.779 INFO:teuthology.orchestra.run.smithi082.stdout:mon.b: {} 2023-10-09T05:00:05.798 INFO:teuthology.orchestra.run.smithi082.stdout:mon.c: {} 2023-10-09T05:00:05.801 INFO:teuthology.orchestra.run.smithi082.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:00:05.801 INFO:teuthology.orchestra.run.smithi082.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:00:05.801 INFO:teuthology.orchestra.run.smithi082.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:00:06.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:06 smithi153 bash[59565]: cluster 2023-10-09T05:00:04.985265+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v1560: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:06.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:06 smithi153 bash[59565]: audit 2023-10-09T05:00:05.107182+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:06.416 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:06 smithi153 bash[59565]: audit 2023-10-09T05:00:05.743234+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.82:0/488419915' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:00:06.417 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[79733]: cluster 2023-10-09T05:00:04.985265+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v1560: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[79733]: audit 2023-10-09T05:00:05.107182+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[79733]: audit 2023-10-09T05:00:05.743234+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.82:0/488419915' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[77063]: cluster 2023-10-09T05:00:04.985265+0000 mgr.y (mgr.34107) 1386 : cluster 0 pgmap v1560: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 409 KiB/s, 2 objects/s recovering 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[77063]: audit 2023-10-09T05:00:05.107182+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:06 smithi082 bash[77063]: audit 2023-10-09T05:00:05.743234+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.82:0/488419915' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:00:06.662 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:00:08.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:08 smithi153 bash[59565]: cluster 2023-10-09T05:00:06.986153+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:08.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:08 smithi082 bash[77063]: cluster 2023-10-09T05:00:06.986153+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:08.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:08 smithi082 bash[79733]: cluster 2023-10-09T05:00:06.986153+0000 mgr.y (mgr.34107) 1387 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:09] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.33.4" 2023-10-09T05:00:10.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:10 smithi153 bash[59565]: cluster 2023-10-09T05:00:08.987021+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v1562: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:10.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:10 smithi082 bash[77063]: cluster 2023-10-09T05:00:08.987021+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v1562: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:10.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:10 smithi082 bash[79733]: cluster 2023-10-09T05:00:08.987021+0000 mgr.y (mgr.34107) 1388 : cluster 0 pgmap v1562: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:11.201 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:11.447 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:12.490 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:12 smithi153 bash[59565]: cluster 2023-10-09T05:00:10.988022+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v1563: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:12.585 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:12 smithi082 bash[77063]: cluster 2023-10-09T05:00:10.988022+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v1563: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:12.585 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:12 smithi082 bash[79733]: cluster 2023-10-09T05:00:10.988022+0000 mgr.y (mgr.34107) 1389 : cluster 0 pgmap v1563: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:14.400 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:14.400 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:14.423 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:14 smithi153 bash[59565]: cluster 2023-10-09T05:00:12.988766+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v1564: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:14.625 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:14 smithi082 bash[77063]: cluster 2023-10-09T05:00:12.988766+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v1564: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:14.625 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:14 smithi082 bash[79733]: cluster 2023-10-09T05:00:12.988766+0000 mgr.y (mgr.34107) 1390 : cluster 0 pgmap v1564: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:15.235 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:00:15.260 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:00:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:15 smithi153 bash[59565]: audit 2023-10-09T05:00:14.311168+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:15 smithi153 bash[59565]: audit 2023-10-09T05:00:15.081741+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:15 smithi082 bash[77063]: audit 2023-10-09T05:00:14.311168+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:15.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:15 smithi082 bash[77063]: audit 2023-10-09T05:00:15.081741+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:15.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:15 smithi082 bash[79733]: audit 2023-10-09T05:00:14.311168+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:15.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:15 smithi082 bash[79733]: audit 2023-10-09T05:00:15.081741+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:16.208 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:14.989642+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v1565: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.250725+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: audit 2023-10-09T05:00:15.255049+0000 mon.a (mon.0) 2015 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.255130+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.565061+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.567651+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.851268+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[79733]: cluster 2023-10-09T05:00:15.854136+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:14.989642+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v1565: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.250725+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: audit 2023-10-09T05:00:15.255049+0000 mon.a (mon.0) 2015 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.255130+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.565061+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.567651+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.851268+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:16 smithi082 bash[77063]: cluster 2023-10-09T05:00:15.854136+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:14.989642+0000 mgr.y (mgr.34107) 1391 : cluster 0 pgmap v1565: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.250725+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: audit 2023-10-09T05:00:15.255049+0000 mon.a (mon.0) 2015 : audit 1 from='client.? 172.21.15.82:0/3612923730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.255130+0000 mon.a (mon.0) 2016 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:00:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.565061+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.567651+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.851268+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:16.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:16 smithi153 bash[59565]: cluster 2023-10-09T05:00:15.854136+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T05:00:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:18 smithi153 bash[59565]: cluster 2023-10-09T05:00:16.990543+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v1567: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:18.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:18 smithi082 bash[77063]: cluster 2023-10-09T05:00:16.990543+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v1567: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:18.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:18 smithi082 bash[79733]: cluster 2023-10-09T05:00:16.990543+0000 mgr.y (mgr.34107) 1392 : cluster 0 pgmap v1567: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:19.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:19] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:00:20.009 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:20 smithi153 bash[59565]: cluster 2023-10-09T05:00:18.991255+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v1568: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:20.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:20 smithi082 bash[77063]: cluster 2023-10-09T05:00:18.991255+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v1568: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:20.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:20 smithi082 bash[79733]: cluster 2023-10-09T05:00:18.991255+0000 mgr.y (mgr.34107) 1393 : cluster 0 pgmap v1568: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:20.996 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:22.432 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:22 smithi082 bash[77063]: cluster 2023-10-09T05:00:20.992167+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:22.432 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:22 smithi082 bash[79733]: cluster 2023-10-09T05:00:20.992167+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:22.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:22 smithi153 bash[59565]: cluster 2023-10-09T05:00:20.992167+0000 mgr.y (mgr.34107) 1394 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:23.342 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:23.352 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:23.369 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:23 smithi153 bash[59565]: audit 2023-10-09T05:00:23.016863+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:23.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:23 smithi082 bash[79733]: audit 2023-10-09T05:00:23.016863+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:23.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:23 smithi082 bash[77063]: audit 2023-10-09T05:00:23.016863+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:00:24.156 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:00:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:24 smithi153 bash[59565]: cluster 2023-10-09T05:00:22.992997+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:24 smithi082 bash[77063]: cluster 2023-10-09T05:00:22.992997+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:24 smithi082 bash[79733]: cluster 2023-10-09T05:00:22.992997+0000 mgr.y (mgr.34107) 1395 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:25.299 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T05:00:25.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:25 smithi153 bash[59565]: audit 2023-10-09T05:00:24.600777+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.82:0/2816290216' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:25 smithi153 bash[59565]: audit 2023-10-09T05:00:24.601613+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:25 smithi153 bash[59565]: cluster 2023-10-09T05:00:24.993808+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[77063]: audit 2023-10-09T05:00:24.600777+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.82:0/2816290216' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[77063]: audit 2023-10-09T05:00:24.601613+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[77063]: cluster 2023-10-09T05:00:24.993808+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:25.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[79733]: audit 2023-10-09T05:00:24.600777+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.82:0/2816290216' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[79733]: audit 2023-10-09T05:00:24.601613+0000 mon.a (mon.0) 2018 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:00:25.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:25 smithi082 bash[79733]: cluster 2023-10-09T05:00:24.993808+0000 mgr.y (mgr.34107) 1396 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:26.226 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:00:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[77063]: cluster 2023-10-09T05:00:25.284776+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[77063]: audit 2023-10-09T05:00:25.291108+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[77063]: cluster 2023-10-09T05:00:25.291265+0000 mon.a (mon.0) 2020 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:00:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[77063]: cluster 2023-10-09T05:00:25.570779+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[77063]: cluster 2023-10-09T05:00:25.573702+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[79733]: cluster 2023-10-09T05:00:25.284776+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[79733]: audit 2023-10-09T05:00:25.291108+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[79733]: cluster 2023-10-09T05:00:25.291265+0000 mon.a (mon.0) 2020 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:00:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[79733]: cluster 2023-10-09T05:00:25.570779+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:26 smithi082 bash[79733]: cluster 2023-10-09T05:00:25.573702+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:26 smithi153 bash[59565]: cluster 2023-10-09T05:00:25.284776+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:26 smithi153 bash[59565]: audit 2023-10-09T05:00:25.291108+0000 mon.a (mon.0) 2019 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:00:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:26 smithi153 bash[59565]: cluster 2023-10-09T05:00:25.291265+0000 mon.a (mon.0) 2020 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:00:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:26 smithi153 bash[59565]: cluster 2023-10-09T05:00:25.570779+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:26.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:26 smithi153 bash[59565]: cluster 2023-10-09T05:00:25.573702+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T05:00:27.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:27 smithi153 bash[59565]: cluster 2023-10-09T05:00:26.994661+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:27.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:27 smithi082 bash[77063]: cluster 2023-10-09T05:00:26.994661+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:27.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:27 smithi082 bash[79733]: cluster 2023-10-09T05:00:26.994661+0000 mgr.y (mgr.34107) 1397 : cluster 0 pgmap v1573: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:28.917 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:29] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: cluster 2023-10-09T05:00:28.995446+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: audit 2023-10-09T05:00:29.208978+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: audit 2023-10-09T05:00:29.217147+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: audit 2023-10-09T05:00:29.311675+0000 mon.a (mon.0) 2023 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: audit 2023-10-09T05:00:29.975451+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.358 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:30 smithi153 bash[59565]: audit 2023-10-09T05:00:29.983766+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: cluster 2023-10-09T05:00:28.995446+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: audit 2023-10-09T05:00:29.208978+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: audit 2023-10-09T05:00:29.217147+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: audit 2023-10-09T05:00:29.311675+0000 mon.a (mon.0) 2023 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: audit 2023-10-09T05:00:29.975451+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[77063]: audit 2023-10-09T05:00:29.983766+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: cluster 2023-10-09T05:00:28.995446+0000 mgr.y (mgr.34107) 1398 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: audit 2023-10-09T05:00:29.208978+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: audit 2023-10-09T05:00:29.217147+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: audit 2023-10-09T05:00:29.311675+0000 mon.a (mon.0) 2023 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: audit 2023-10-09T05:00:29.975451+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:30 smithi082 bash[79733]: audit 2023-10-09T05:00:29.983766+0000 mon.a (mon.0) 2025 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:30.777 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:31.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:31 smithi153 bash[59565]: audit 2023-10-09T05:00:30.455594+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:31.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:31 smithi153 bash[59565]: audit 2023-10-09T05:00:30.456894+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:31.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:31 smithi153 bash[59565]: audit 2023-10-09T05:00:30.464623+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:31.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[79733]: audit 2023-10-09T05:00:30.455594+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:31.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[79733]: audit 2023-10-09T05:00:30.456894+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:31.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[79733]: audit 2023-10-09T05:00:30.464623+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:31.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[77063]: audit 2023-10-09T05:00:30.455594+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:31.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[77063]: audit 2023-10-09T05:00:30.456894+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:31.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:31 smithi082 bash[77063]: audit 2023-10-09T05:00:30.464623+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:00:32.006 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:32.006 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:32.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:32 smithi153 bash[59565]: cluster 2023-10-09T05:00:30.996071+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v1575: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:32.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:32 smithi082 bash[79733]: cluster 2023-10-09T05:00:30.996071+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v1575: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:32.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:32 smithi082 bash[77063]: cluster 2023-10-09T05:00:30.996071+0000 mgr.y (mgr.34107) 1399 : cluster 0 pgmap v1575: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:32.860 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:00:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:34 smithi082 bash[77063]: cluster 2023-10-09T05:00:32.996839+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:34 smithi082 bash[79733]: cluster 2023-10-09T05:00:32.996839+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:34.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:34 smithi153 bash[59565]: cluster 2023-10-09T05:00:32.996839+0000 mgr.y (mgr.34107) 1400 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:35.267 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:00:35.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:35 smithi153 bash[59565]: audit 2023-10-09T05:00:34.598371+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.82:0/609320874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:35.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:35 smithi153 bash[59565]: audit 2023-10-09T05:00:34.599043+0000 mon.a (mon.0) 2029 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:35 smithi082 bash[77063]: audit 2023-10-09T05:00:34.598371+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.82:0/609320874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:35.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:35 smithi082 bash[77063]: audit 2023-10-09T05:00:34.599043+0000 mon.a (mon.0) 2029 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:35.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:35 smithi082 bash[79733]: audit 2023-10-09T05:00:34.598371+0000 mon.c (mon.2) 151 : audit 1 from='client.? 172.21.15.82:0/609320874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:35.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:35 smithi082 bash[79733]: audit 2023-10-09T05:00:34.599043+0000 mon.a (mon.0) 2029 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:36.165 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T05:00:36.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: cluster 2023-10-09T05:00:34.997631+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: cluster 2023-10-09T05:00:35.253056+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: audit 2023-10-09T05:00:35.261904+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: cluster 2023-10-09T05:00:35.262011+0000 mon.a (mon.0) 2031 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: cluster 2023-10-09T05:00:35.578152+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[77063]: cluster 2023-10-09T05:00:36.057357+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: cluster 2023-10-09T05:00:34.997631+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:36.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: cluster 2023-10-09T05:00:35.253056+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: audit 2023-10-09T05:00:35.261904+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: cluster 2023-10-09T05:00:35.262011+0000 mon.a (mon.0) 2031 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:00:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: cluster 2023-10-09T05:00:35.578152+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.483 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:36 smithi082 bash[79733]: cluster 2023-10-09T05:00:36.057357+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: cluster 2023-10-09T05:00:34.997631+0000 mgr.y (mgr.34107) 1401 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: cluster 2023-10-09T05:00:35.253056+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: audit 2023-10-09T05:00:35.261904+0000 mon.a (mon.0) 2030 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: cluster 2023-10-09T05:00:35.262011+0000 mon.a (mon.0) 2031 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: cluster 2023-10-09T05:00:35.578152+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:36.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:36 smithi153 bash[59565]: cluster 2023-10-09T05:00:36.057357+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:37.630 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:37.660 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:37 smithi153 bash[59565]: cluster 2023-10-09T05:00:35.652394+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:37.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:37 smithi082 bash[77063]: cluster 2023-10-09T05:00:35.652394+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:37.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:37 smithi082 bash[79733]: cluster 2023-10-09T05:00:35.652394+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T05:00:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:38 smithi153 bash[59565]: cluster 2023-10-09T05:00:36.998560+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:38.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:38 smithi082 bash[77063]: cluster 2023-10-09T05:00:36.998560+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:38.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:38 smithi082 bash[79733]: cluster 2023-10-09T05:00:36.998560+0000 mgr.y (mgr.34107) 1402 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:39] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:00:40.526 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:40 smithi153 bash[59565]: cluster 2023-10-09T05:00:38.999476+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:40.705 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:40.706 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:40.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:40 smithi082 bash[77063]: cluster 2023-10-09T05:00:38.999476+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:40.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:40 smithi082 bash[79733]: cluster 2023-10-09T05:00:38.999476+0000 mgr.y (mgr.34107) 1403 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:40.945 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:41.547 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:00:42.451 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:42 smithi082 bash[77063]: cluster 2023-10-09T05:00:41.000445+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:42.452 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:42 smithi082 bash[79733]: cluster 2023-10-09T05:00:41.000445+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:42.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:42 smithi153 bash[59565]: cluster 2023-10-09T05:00:41.000445+0000 mgr.y (mgr.34107) 1404 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:43.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:43 smithi082 bash[79733]: cluster 2023-10-09T05:00:43.001390+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:43.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:43 smithi082 bash[77063]: cluster 2023-10-09T05:00:43.001390+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:43.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:43 smithi153 bash[59565]: cluster 2023-10-09T05:00:43.001390+0000 mgr.y (mgr.34107) 1405 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:44 smithi153 bash[59565]: audit 2023-10-09T05:00:44.312338+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:44 smithi082 bash[77063]: audit 2023-10-09T05:00:44.312338+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:44 smithi082 bash[79733]: audit 2023-10-09T05:00:44.312338+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:45.385 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T05:00:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:45 smithi153 bash[59565]: audit 2023-10-09T05:00:44.721430+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.82:0/2152865112' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:45 smithi153 bash[59565]: audit 2023-10-09T05:00:44.722122+0000 mon.a (mon.0) 2033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:45 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.002098+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[79733]: audit 2023-10-09T05:00:44.721430+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.82:0/2152865112' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[79733]: audit 2023-10-09T05:00:44.722122+0000 mon.a (mon.0) 2033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.002098+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[77063]: audit 2023-10-09T05:00:44.721430+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.82:0/2152865112' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[77063]: audit 2023-10-09T05:00:44.722122+0000 mon.a (mon.0) 2033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:00:45.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:45 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.002098+0000 mgr.y (mgr.34107) 1406 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:46.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:00:46.330 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.375434+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.375766+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: audit 2023-10-09T05:00:45.377529+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:46.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.377651+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:00:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.381159+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.381250+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:46 smithi153 bash[59565]: cluster 2023-10-09T05:00:45.384131+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.375434+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.375766+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: audit 2023-10-09T05:00:45.377529+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.377651+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:00:46.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.381159+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.381250+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[77063]: cluster 2023-10-09T05:00:45.384131+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.375434+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.375766+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: audit 2023-10-09T05:00:45.377529+0000 mon.a (mon.0) 2034 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.377651+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.381159+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.381250+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:46.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:46 smithi082 bash[79733]: cluster 2023-10-09T05:00:45.384131+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T05:00:47.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:46.385907+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:46.386330+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.655 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:46.388732+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:00:47.656 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:46.391359+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.656 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:46.392426+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.656 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:47 smithi153 bash[59565]: cluster 2023-10-09T05:00:47.002762+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v1586: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:46.385907+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:46.386330+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:46.388732+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:46.391359+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:46.392426+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[77063]: cluster 2023-10-09T05:00:47.002762+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v1586: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:46.385907+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:46.386330+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:46.388732+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:46.391359+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:46.392426+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T05:00:47.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:47 smithi082 bash[79733]: cluster 2023-10-09T05:00:47.002762+0000 mgr.y (mgr.34107) 1407 : cluster 0 pgmap v1586: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:49.547 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:49.547 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:49] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:00:50.386 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:50 smithi153 bash[59565]: cluster 2023-10-09T05:00:49.003568+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v1587: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:50.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:00:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:50 smithi082 bash[77063]: cluster 2023-10-09T05:00:49.003568+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v1587: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:50 smithi082 bash[79733]: cluster 2023-10-09T05:00:49.003568+0000 mgr.y (mgr.34107) 1408 : cluster 0 pgmap v1587: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:51.091 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:52.460 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:52 smithi082 bash[77063]: cluster 2023-10-09T05:00:51.004676+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v1588: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:52.460 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:52 smithi082 bash[79733]: cluster 2023-10-09T05:00:51.004676+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v1588: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:52.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:52 smithi153 bash[59565]: cluster 2023-10-09T05:00:51.004676+0000 mgr.y (mgr.34107) 1409 : cluster 0 pgmap v1588: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:54.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:54 smithi153 bash[59565]: cluster 2023-10-09T05:00:53.005271+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v1589: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:54.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:54 smithi082 bash[77063]: cluster 2023-10-09T05:00:53.005271+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v1589: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:54.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:54 smithi082 bash[79733]: cluster 2023-10-09T05:00:53.005271+0000 mgr.y (mgr.34107) 1410 : cluster 0 pgmap v1589: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:55.165 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:00:55.267 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:00:55.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:55 smithi153 bash[59565]: audit 2023-10-09T05:00:54.714390+0000 mon.a (mon.0) 2037 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:00:55.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:55 smithi082 bash[77063]: audit 2023-10-09T05:00:54.714390+0000 mon.a (mon.0) 2037 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:00:55.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:55 smithi082 bash[79733]: audit 2023-10-09T05:00:54.714390+0000 mon.a (mon.0) 2037 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:00:56.168 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T05:00:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[79733]: cluster 2023-10-09T05:00:55.006061+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v1590: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[79733]: cluster 2023-10-09T05:00:55.259143+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:00:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[79733]: audit 2023-10-09T05:00:55.260432+0000 mon.a (mon.0) 2038 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:00:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[79733]: cluster 2023-10-09T05:00:55.260519+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:00:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[77063]: cluster 2023-10-09T05:00:55.006061+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v1590: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[77063]: cluster 2023-10-09T05:00:55.259143+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:00:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[77063]: audit 2023-10-09T05:00:55.260432+0000 mon.a (mon.0) 2038 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:00:56.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:56 smithi082 bash[77063]: cluster 2023-10-09T05:00:55.260519+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:00:56.535 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:56 smithi153 bash[59565]: cluster 2023-10-09T05:00:55.006061+0000 mgr.y (mgr.34107) 1411 : cluster 0 pgmap v1590: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:56.536 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:56 smithi153 bash[59565]: cluster 2023-10-09T05:00:55.259143+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T05:00:56.536 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:56 smithi153 bash[59565]: audit 2023-10-09T05:00:55.260432+0000 mon.a (mon.0) 2038 : audit 1 from='client.? 172.21.15.82:0/3896829983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:00:56.536 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:56 smithi153 bash[59565]: cluster 2023-10-09T05:00:55.260519+0000 mon.a (mon.0) 2039 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:00:58.294 INFO:teuthology.orchestra.run.smithi153.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:58.295 INFO:teuthology.orchestra.run.smithi153.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:58.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:00:58 smithi153 bash[59565]: cluster 2023-10-09T05:00:57.006883+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v1592: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:00:58 smithi082 bash[77063]: cluster 2023-10-09T05:00:57.006883+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v1592: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:58.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:00:58 smithi082 bash[79733]: cluster 2023-10-09T05:00:57.006883+0000 mgr.y (mgr.34107) 1412 : cluster 0 pgmap v1592: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:00:59.080 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:00:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:00:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:00:59] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:01:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:00 smithi153 bash[59565]: cluster 2023-10-09T05:00:59.007786+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:00 smithi153 bash[59565]: audit 2023-10-09T05:00:59.311712+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:00 smithi082 bash[77063]: cluster 2023-10-09T05:00:59.007786+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:00 smithi082 bash[77063]: audit 2023-10-09T05:00:59.311712+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:00.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:00 smithi082 bash[79733]: cluster 2023-10-09T05:00:59.007786+0000 mgr.y (mgr.34107) 1413 : cluster 0 pgmap v1593: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:00 smithi082 bash[79733]: audit 2023-10-09T05:00:59.311712+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:00.946 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:02.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:02 smithi082 bash[77063]: cluster 2023-10-09T05:01:01.008716+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:02.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:02 smithi082 bash[79733]: cluster 2023-10-09T05:01:01.008716+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:02.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:02 smithi153 bash[59565]: cluster 2023-10-09T05:01:01.008716+0000 mgr.y (mgr.34107) 1414 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:03.686 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:04.567 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:04 smithi082 bash[77063]: cluster 2023-10-09T05:01:03.009496+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:04.567 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:04 smithi082 bash[79733]: cluster 2023-10-09T05:01:03.009496+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:04.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:04 smithi153 bash[59565]: cluster 2023-10-09T05:01:03.009496+0000 mgr.y (mgr.34107) 1415 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:05.300 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T05:01:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:05 smithi153 bash[59565]: audit 2023-10-09T05:01:05.034679+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.82:0/3504737811' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:05.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:05 smithi153 bash[59565]: audit 2023-10-09T05:01:05.035103+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:05.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:05 smithi082 bash[77063]: audit 2023-10-09T05:01:05.034679+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.82:0/3504737811' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:05.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:05 smithi082 bash[77063]: audit 2023-10-09T05:01:05.035103+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:05.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:05 smithi082 bash[79733]: audit 2023-10-09T05:01:05.034679+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.82:0/3504737811' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:05.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:05 smithi082 bash[79733]: audit 2023-10-09T05:01:05.035103+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:01:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.010275+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.283501+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: audit 2023-10-09T05:01:05.286193+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:01:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.286300+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:01:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.286902+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.290366+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.290430+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:06 smithi153 bash[59565]: cluster 2023-10-09T05:01:05.291507+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.010275+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.283501+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: audit 2023-10-09T05:01:05.286193+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:01:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.286300+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:01:06.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.286902+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.290366+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.290430+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[77063]: cluster 2023-10-09T05:01:05.291507+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.010275+0000 mgr.y (mgr.34107) 1416 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.283501+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: audit 2023-10-09T05:01:05.286193+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.286300+0000 mon.a (mon.0) 2043 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.286902+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.290366+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.290430+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:06.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:06 smithi082 bash[79733]: cluster 2023-10-09T05:01:05.291507+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T05:01:07.229 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:01:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[77063]: cluster 2023-10-09T05:01:06.287622+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[77063]: cluster 2023-10-09T05:01:06.287992+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[77063]: cluster 2023-10-09T05:01:06.290270+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:01:07.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[77063]: cluster 2023-10-09T05:01:06.293172+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[77063]: cluster 2023-10-09T05:01:07.010911+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v1599: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[79733]: cluster 2023-10-09T05:01:06.287622+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[79733]: cluster 2023-10-09T05:01:06.287992+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[79733]: cluster 2023-10-09T05:01:06.290270+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[79733]: cluster 2023-10-09T05:01:06.293172+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:07 smithi082 bash[79733]: cluster 2023-10-09T05:01:07.010911+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v1599: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:07.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:07 smithi153 bash[59565]: cluster 2023-10-09T05:01:06.287622+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:07 smithi153 bash[59565]: cluster 2023-10-09T05:01:06.287992+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:07 smithi153 bash[59565]: cluster 2023-10-09T05:01:06.290270+0000 mon.a (mon.0) 2044 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:01:07.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:07 smithi153 bash[59565]: cluster 2023-10-09T05:01:06.293172+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:07.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:07 smithi153 bash[59565]: cluster 2023-10-09T05:01:07.010911+0000 mgr.y (mgr.34107) 1417 : cluster 0 pgmap v1599: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:07.798 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:07.799 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:08.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:08 smithi153 bash[59565]: cluster 2023-10-09T05:01:06.293045+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:08.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:08 smithi082 bash[77063]: cluster 2023-10-09T05:01:06.293045+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:08.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:08 smithi082 bash[79733]: cluster 2023-10-09T05:01:06.293045+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T05:01:08.904 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:01:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:09 smithi153 bash[59565]: cluster 2023-10-09T05:01:09.011460+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:09.699 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:09 smithi082 bash[79733]: cluster 2023-10-09T05:01:09.011460+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:09.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:09 smithi082 bash[77063]: cluster 2023-10-09T05:01:09.011460+0000 mgr.y (mgr.34107) 1418 : cluster 0 pgmap v1600: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:09] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:01:12.441 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:12.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:12 smithi153 bash[59565]: cluster 2023-10-09T05:01:11.012573+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:12.733 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:12 smithi082 bash[77063]: cluster 2023-10-09T05:01:11.012573+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:12.734 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:12 smithi082 bash[79733]: cluster 2023-10-09T05:01:11.012573+0000 mgr.y (mgr.34107) 1419 : cluster 0 pgmap v1601: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:13.458 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:14.614 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:14 smithi082 bash[77063]: cluster 2023-10-09T05:01:13.013335+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:14.614 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:14 smithi082 bash[79733]: cluster 2023-10-09T05:01:13.013335+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:14.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:14 smithi153 bash[59565]: cluster 2023-10-09T05:01:13.013335+0000 mgr.y (mgr.34107) 1420 : cluster 0 pgmap v1602: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:15.555 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:15 smithi082 bash[79733]: audit 2023-10-09T05:01:14.312089+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:15.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:15 smithi082 bash[77063]: audit 2023-10-09T05:01:14.312089+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:15 smithi153 bash[59565]: audit 2023-10-09T05:01:14.312089+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:16.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:16 smithi082 bash[77063]: cluster 2023-10-09T05:01:15.014157+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:16 smithi082 bash[79733]: cluster 2023-10-09T05:01:15.014157+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:16 smithi153 bash[59565]: cluster 2023-10-09T05:01:15.014157+0000 mgr.y (mgr.34107) 1421 : cluster 0 pgmap v1603: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:18.281 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:01:18.644 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[77063]: cluster 2023-10-09T05:01:17.014686+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v1604: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:18.644 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[77063]: audit 2023-10-09T05:01:18.076473+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.82:0/2757274394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:18.644 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[77063]: audit 2023-10-09T05:01:18.076991+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:18.644 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[79733]: cluster 2023-10-09T05:01:17.014686+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v1604: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:18.645 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[79733]: audit 2023-10-09T05:01:18.076473+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.82:0/2757274394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:18.645 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:18 smithi082 bash[79733]: audit 2023-10-09T05:01:18.076991+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:18 smithi153 bash[59565]: cluster 2023-10-09T05:01:17.014686+0000 mgr.y (mgr.34107) 1422 : cluster 0 pgmap v1604: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:18 smithi153 bash[59565]: audit 2023-10-09T05:01:18.076473+0000 mon.c (mon.2) 154 : audit 1 from='client.? 172.21.15.82:0/2757274394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:18.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:18 smithi153 bash[59565]: audit 2023-10-09T05:01:18.076991+0000 mon.a (mon.0) 2046 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:19.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:19 smithi153 bash[59565]: cluster 2023-10-09T05:01:18.271732+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:19 smithi153 bash[59565]: audit 2023-10-09T05:01:18.275798+0000 mon.a (mon.0) 2047 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:19.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:19 smithi153 bash[59565]: cluster 2023-10-09T05:01:18.275919+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T05:01:19.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:19 smithi153 bash[59565]: cluster 2023-10-09T05:01:18.359213+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:19 smithi153 bash[59565]: cluster 2023-10-09T05:01:18.525906+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[77063]: cluster 2023-10-09T05:01:18.271732+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[77063]: audit 2023-10-09T05:01:18.275798+0000 mon.a (mon.0) 2047 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:19.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[77063]: cluster 2023-10-09T05:01:18.275919+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T05:01:19.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[77063]: cluster 2023-10-09T05:01:18.359213+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.701 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[77063]: cluster 2023-10-09T05:01:18.525906+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[79733]: cluster 2023-10-09T05:01:18.271732+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[79733]: audit 2023-10-09T05:01:18.275798+0000 mon.a (mon.0) 2047 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:19.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[79733]: cluster 2023-10-09T05:01:18.275919+0000 mon.a (mon.0) 2048 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T05:01:19.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[79733]: cluster 2023-10-09T05:01:18.359213+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.702 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[79733]: cluster 2023-10-09T05:01:18.525906+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:19.718 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T05:01:19.938 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:19.939 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:19.959 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:19] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T05:01:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:20 smithi082 bash[77063]: cluster 2023-10-09T05:01:18.683141+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:20 smithi082 bash[77063]: cluster 2023-10-09T05:01:19.015175+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:20.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:20 smithi082 bash[79733]: cluster 2023-10-09T05:01:18.683141+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:20.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:20 smithi082 bash[79733]: cluster 2023-10-09T05:01:19.015175+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:20 smithi153 bash[59565]: cluster 2023-10-09T05:01:18.683141+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T05:01:20.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:20 smithi153 bash[59565]: cluster 2023-10-09T05:01:19.015175+0000 mgr.y (mgr.34107) 1423 : cluster 0 pgmap v1606: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:21.056 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:01:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:21 smithi082 bash[77063]: cluster 2023-10-09T05:01:21.016003+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:21 smithi082 bash[79733]: cluster 2023-10-09T05:01:21.016003+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:21 smithi153 bash[59565]: cluster 2023-10-09T05:01:21.016003+0000 mgr.y (mgr.34107) 1424 : cluster 0 pgmap v1607: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:24.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:24 smithi153 bash[59565]: cluster 2023-10-09T05:01:23.016691+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:24 smithi082 bash[77063]: cluster 2023-10-09T05:01:23.016691+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:24 smithi082 bash[79733]: cluster 2023-10-09T05:01:23.016691+0000 mgr.y (mgr.34107) 1425 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:24.585 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:25.599 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:26.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:26 smithi153 bash[59565]: cluster 2023-10-09T05:01:25.017332+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:26 smithi082 bash[77063]: cluster 2023-10-09T05:01:25.017332+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:26.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:26 smithi082 bash[79733]: cluster 2023-10-09T05:01:25.017332+0000 mgr.y (mgr.34107) 1426 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:28.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:28 smithi153 bash[59565]: cluster 2023-10-09T05:01:27.018083+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:28.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:28 smithi082 bash[77063]: cluster 2023-10-09T05:01:27.018083+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:28.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:28 smithi082 bash[79733]: cluster 2023-10-09T05:01:27.018083+0000 mgr.y (mgr.34107) 1427 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:29.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:29] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-10-09T05:01:30.285 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T05:01:30.617 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:30.617 INFO:teuthology.orchestra.run.smithi082.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:30.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:30 smithi153 bash[59565]: cluster 2023-10-09T05:01:29.018807+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:30.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:30 smithi153 bash[59565]: audit 2023-10-09T05:01:29.312834+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:30.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:30 smithi153 bash[59565]: audit 2023-10-09T05:01:29.812024+0000 mon.a (mon.0) 2050 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[77063]: cluster 2023-10-09T05:01:29.018807+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[77063]: audit 2023-10-09T05:01:29.312834+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[77063]: audit 2023-10-09T05:01:29.812024+0000 mon.a (mon.0) 2050 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[79733]: cluster 2023-10-09T05:01:29.018807+0000 mgr.y (mgr.34107) 1428 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[79733]: audit 2023-10-09T05:01:29.312834+0000 mon.a (mon.0) 2049 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:30.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:30 smithi082 bash[79733]: audit 2023-10-09T05:01:29.812024+0000 mon.a (mon.0) 2050 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:01:31.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: cluster 2023-10-09T05:01:30.274330+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.483 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: cluster 2023-10-09T05:01:30.274777+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: audit 2023-10-09T05:01:30.278946+0000 mon.a (mon.0) 2051 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:31.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: cluster 2023-10-09T05:01:30.279002+0000 mon.a (mon.0) 2052 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T05:01:31.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: audit 2023-10-09T05:01:30.548572+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:31.484 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[77063]: cluster 2023-10-09T05:01:30.785519+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: cluster 2023-10-09T05:01:30.274330+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: cluster 2023-10-09T05:01:30.274777+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.485 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: audit 2023-10-09T05:01:30.278946+0000 mon.a (mon.0) 2051 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:31.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: cluster 2023-10-09T05:01:30.279002+0000 mon.a (mon.0) 2052 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T05:01:31.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: audit 2023-10-09T05:01:30.548572+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:31.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:31 smithi082 bash[79733]: cluster 2023-10-09T05:01:30.785519+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:01:31.880 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: cluster 2023-10-09T05:01:30.274330+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.880 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: cluster 2023-10-09T05:01:30.274777+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:31.880 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: audit 2023-10-09T05:01:30.278946+0000 mon.a (mon.0) 2051 : audit 1 from='client.? 172.21.15.82:0/1250027156' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:01:31.881 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: cluster 2023-10-09T05:01:30.279002+0000 mon.a (mon.0) 2052 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T05:01:31.881 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: audit 2023-10-09T05:01:30.548572+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:31.881 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:31 smithi153 bash[59565]: cluster 2023-10-09T05:01:30.785519+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:32.347 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:01:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:32 smithi153 bash[59565]: cluster 2023-10-09T05:01:30.616158+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:32.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:32 smithi153 bash[59565]: cluster 2023-10-09T05:01:31.019581+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:32 smithi082 bash[77063]: cluster 2023-10-09T05:01:30.616158+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:32.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:32 smithi082 bash[77063]: cluster 2023-10-09T05:01:31.019581+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:32.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:32 smithi082 bash[79733]: cluster 2023-10-09T05:01:30.616158+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T05:01:32.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:32 smithi082 bash[79733]: cluster 2023-10-09T05:01:31.019581+0000 mgr.y (mgr.34107) 1429 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:33.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:33 smithi153 bash[59565]: cluster 2023-10-09T05:01:33.020342+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:33.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:33 smithi082 bash[77063]: cluster 2023-10-09T05:01:33.020342+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:33.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:33 smithi082 bash[79733]: cluster 2023-10-09T05:01:33.020342+0000 mgr.y (mgr.34107) 1430 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:35.407 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:36.443 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:36.523 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:36 smithi082 bash[79733]: cluster 2023-10-09T05:01:35.021162+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:36.524 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:36 smithi082 bash[77063]: cluster 2023-10-09T05:01:35.021162+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:36.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:36 smithi153 bash[59565]: cluster 2023-10-09T05:01:35.021162+0000 mgr.y (mgr.34107) 1431 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:37.839 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:36.537764+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.839 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:36.546855+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:36.739563+0000 mon.a (mon.0) 2056 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:36.748202+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: cluster 2023-10-09T05:01:37.021687+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v1616: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:37.222756+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:37.224301+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:37.840 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:37 smithi153 bash[59565]: audit 2023-10-09T05:01:37.236566+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:36.537764+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:36.546855+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:36.739563+0000 mon.a (mon.0) 2056 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:36.748202+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: cluster 2023-10-09T05:01:37.021687+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v1616: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:37.222756+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:37.224301+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[77063]: audit 2023-10-09T05:01:37.236566+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:36.537764+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:36.546855+0000 mon.a (mon.0) 2055 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:36.739563+0000 mon.a (mon.0) 2056 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:36.748202+0000 mon.a (mon.0) 2057 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:37.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: cluster 2023-10-09T05:01:37.021687+0000 mgr.y (mgr.34107) 1432 : cluster 0 pgmap v1616: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:37.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:37.222756+0000 mon.a (mon.0) 2058 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:37.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:37.224301+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:37.983 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:37 smithi082 bash[79733]: audit 2023-10-09T05:01:37.236566+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:01:39.091 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:01:39.340 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:39 smithi153 bash[59565]: audit 2023-10-09T05:01:39.026841+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:39.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:39 smithi082 bash[79733]: audit 2023-10-09T05:01:39.026841+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:39.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:39 smithi082 bash[77063]: audit 2023-10-09T05:01:39.026841+0000 mon.a (mon.0) 2061 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:39.553 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T05:01:39.553 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T05:01:39.553 INFO:teuthology.orchestra.run.smithi153.stdout: "num_ops": 0 2023-10-09T05:01:39.553 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T05:01:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:39] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-10-09T05:01:40.042 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T05:01:40.337 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:01:40.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:40 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.022493+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:40.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:40 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.086118+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:40.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:40 smithi153 bash[59565]: audit 2023-10-09T05:01:39.086200+0000 mon.a (mon.0) 2062 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:40.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:40 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.086291+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T05:01:40.348 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:40 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.525679+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.022493+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.086118+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[77063]: audit 2023-10-09T05:01:39.086200+0000 mon.a (mon.0) 2062 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.086291+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.525679+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.022493+0000 mgr.y (mgr.34107) 1433 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.086118+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[79733]: audit 2023-10-09T05:01:39.086200+0000 mon.a (mon.0) 2062 : audit 1 from='client.? 172.21.15.82:0/3520001283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.086291+0000 mon.a (mon.0) 2063 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T05:01:40.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:40 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.525679+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:41 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.587085+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:41 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.589731+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:41 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.629941+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:41 smithi153 bash[59565]: cluster 2023-10-09T05:01:39.633175+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.587085+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.589731+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.629941+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[77063]: cluster 2023-10-09T05:01:39.633175+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.587085+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.589731+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.629941+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:41.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:41 smithi082 bash[79733]: cluster 2023-10-09T05:01:39.633175+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T05:01:42.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:42 smithi153 bash[59565]: cluster 2023-10-09T05:01:41.023571+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v1619: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:42 smithi082 bash[77063]: cluster 2023-10-09T05:01:41.023571+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v1619: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:42 smithi082 bash[79733]: cluster 2023-10-09T05:01:41.023571+0000 mgr.y (mgr.34107) 1434 : cluster 0 pgmap v1619: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:44 smithi153 bash[59565]: cluster 2023-10-09T05:01:43.024330+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v1620: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:44.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:44 smithi082 bash[77063]: cluster 2023-10-09T05:01:43.024330+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v1620: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:44 smithi082 bash[79733]: cluster 2023-10-09T05:01:43.024330+0000 mgr.y (mgr.34107) 1435 : cluster 0 pgmap v1620: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:44.822 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:45.116 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:45 smithi153 bash[59565]: audit 2023-10-09T05:01:44.312889+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:45 smithi082 bash[79733]: audit 2023-10-09T05:01:44.312889+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:45 smithi082 bash[77063]: audit 2023-10-09T05:01:44.312889+0000 mon.a (mon.0) 2064 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:46 smithi153 bash[59565]: cluster 2023-10-09T05:01:45.024850+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:46.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:46 smithi082 bash[79733]: cluster 2023-10-09T05:01:45.024850+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:46.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:46 smithi082 bash[77063]: cluster 2023-10-09T05:01:45.024850+0000 mgr.y (mgr.34107) 1436 : cluster 0 pgmap v1621: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:48.134 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T05:01:48.134 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [], 2023-10-09T05:01:48.135 INFO:teuthology.orchestra.run.smithi153.stdout: "complaint_time": 30, 2023-10-09T05:01:48.135 INFO:teuthology.orchestra.run.smithi153.stdout: "num_blocked_ops": 0 2023-10-09T05:01:48.135 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T05:01:48.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:48 smithi082 bash[77063]: cluster 2023-10-09T05:01:47.025706+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v1622: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:48.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:48 smithi082 bash[79733]: cluster 2023-10-09T05:01:47.025706+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v1622: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:48.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:48 smithi153 bash[59565]: cluster 2023-10-09T05:01:47.025706+0000 mgr.y (mgr.34107) 1437 : cluster 0 pgmap v1622: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:48.929 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:01:49.304 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:49 smithi153 bash[59565]: audit 2023-10-09T05:01:48.667635+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.82:0/2659776241' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:49 smithi153 bash[59565]: audit 2023-10-09T05:01:48.668343+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:49 smithi082 bash[77063]: audit 2023-10-09T05:01:48.667635+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.82:0/2659776241' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:49 smithi082 bash[77063]: audit 2023-10-09T05:01:48.668343+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:49 smithi082 bash[79733]: audit 2023-10-09T05:01:48.667635+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.82:0/2659776241' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:49 smithi082 bash[79733]: audit 2023-10-09T05:01:48.668343+0000 mon.a (mon.0) 2065 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:01:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:49] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:01:50.149 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: cluster 2023-10-09T05:01:49.026490+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v1623: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: cluster 2023-10-09T05:01:49.294124+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: cluster 2023-10-09T05:01:49.294996+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: audit 2023-10-09T05:01:49.296546+0000 mon.a (mon.0) 2066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: cluster 2023-10-09T05:01:49.296644+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T05:01:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[79733]: cluster 2023-10-09T05:01:49.989675+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: cluster 2023-10-09T05:01:49.026490+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v1623: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: cluster 2023-10-09T05:01:49.294124+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: cluster 2023-10-09T05:01:49.294996+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: audit 2023-10-09T05:01:49.296546+0000 mon.a (mon.0) 2066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: cluster 2023-10-09T05:01:49.296644+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T05:01:50.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:50 smithi082 bash[77063]: cluster 2023-10-09T05:01:49.989675+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: cluster 2023-10-09T05:01:49.026490+0000 mgr.y (mgr.34107) 1438 : cluster 0 pgmap v1623: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: cluster 2023-10-09T05:01:49.294124+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: cluster 2023-10-09T05:01:49.294996+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: audit 2023-10-09T05:01:49.296546+0000 mon.a (mon.0) 2066 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: cluster 2023-10-09T05:01:49.296644+0000 mon.a (mon.0) 2067 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T05:01:50.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:50 smithi153 bash[59565]: cluster 2023-10-09T05:01:49.989675+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T05:01:51.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:51 smithi153 bash[59565]: cluster 2023-10-09T05:01:51.027442+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v1625: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:51.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:51 smithi082 bash[77063]: cluster 2023-10-09T05:01:51.027442+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v1625: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:51.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:51 smithi082 bash[79733]: cluster 2023-10-09T05:01:51.027442+0000 mgr.y (mgr.34107) 1439 : cluster 0 pgmap v1625: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:53.751 INFO:teuthology.orchestra.run.smithi153.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:54 smithi153 bash[59565]: cluster 2023-10-09T05:01:53.028270+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:54 smithi082 bash[77063]: cluster 2023-10-09T05:01:53.028270+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:54 smithi082 bash[79733]: cluster 2023-10-09T05:01:53.028270+0000 mgr.y (mgr.34107) 1440 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:54.919 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:56.368 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:56 smithi082 bash[79733]: cluster 2023-10-09T05:01:55.029101+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v1627: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:56.440 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:56 smithi082 bash[77063]: cluster 2023-10-09T05:01:55.029101+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v1627: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:56.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:56 smithi153 bash[59565]: cluster 2023-10-09T05:01:55.029101+0000 mgr.y (mgr.34107) 1441 : cluster 0 pgmap v1627: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "size": 20, 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 600, 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "ops": [ 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.7 e359,358 2:e2bf3de7:::rbd_data.b3d264da6c34.00000000000000f4:head,341'35)", 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:47.307203+0000", 2023-10-09T05:01:57.012 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 129.69965616900001, 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047083900000000002, 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307203+0000", 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.013 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307203+0000", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999995 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307203+0000", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1280000000000002e-06 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307210+0000", 2023-10-09T05:01:57.014 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8919999999999995e-06 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307220+0000", 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.5747999999999999e-05 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307236+0000", 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000378783 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.015 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307614+0000", 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6320000000000002e-06 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307621+0000", 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.3292999999999999e-05 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.016 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.307674+0000", 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00047083900000000002 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.7 e359,358 2:f709ef6d:::rbd_object_map.b408797ccaff.0000000000000011:head,343'47)", 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:47.309014+0000", 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 129.69784546599999, 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021488599999999999, 2023-10-09T05:01:57.017 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309014+0000", 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309014+0000", 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.018 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309013+0000", 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.1519999999999992e-06 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309022+0000", 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2825e-05 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.019 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309034+0000", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.3750000000000003e-06 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309043+0000", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00011561399999999999 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309158+0000", 2023-10-09T05:01:57.020 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.435e-06 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309166+0000", 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.3229000000000001e-05 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:47.309229+0000", 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021488599999999999 2023-10-09T05:01:57.021 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.4 e360,359 2:28598933:::rbd_header.8f3741ad5de:head,354'221)", 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.484865+0000", 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.521994294, 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000218462, 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.022 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484865+0000", 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484865+0000", 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.023 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484864+0000", 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7359000000000001e-05 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484881+0000", 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3370000000000002e-05 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484905+0000", 2023-10-09T05:01:57.024 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.269e-05 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484917+0000", 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6941999999999996e-05 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.484994+0000", 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2376e-05 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.025 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.485007+0000", 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7235000000000002e-05 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.485084+0000", 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000218462 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.026 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.027 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.027 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.4 360/359 [PushOp(2:3831950c:::rbd_directory:head, version: 354'230, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:3831950c:::rbd_directory:head@354'230, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.487470+0000", 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.51938933599999, 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025070099999999998, 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487470+0000", 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.028 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487470+0000", 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487469+0000", 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.2183000000000001e-05 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.029 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487491+0000", 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3269999999999999e-05 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487515+0000", 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2012e-05 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.030 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487527+0000", 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.5834000000000001e-05 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487583+0000", 2023-10-09T05:01:57.031 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 5.9869999999999996e-06 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487589+0000", 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000132478 2023-10-09T05:01:57.033 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.487721+0000", 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025070099999999998 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.3 e360,359 2:c9747d68:::rbd_header.b3d264da6c34:head,341'255)", 2023-10-09T05:01:57.034 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.490852+0000", 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.51600794800001, 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020511600000000001, 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490852+0000", 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.035 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490852+0000", 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490851+0000", 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.0989999999999999e-06 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.036 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490859+0000", 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7626e-05 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490876+0000", 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4100000000000008e-06 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490885+0000", 2023-10-09T05:01:57.037 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2725000000000005e-05 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490968+0000", 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1671000000000001e-05 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.490979+0000", 2023-10-09T05:01:57.038 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.7634999999999998e-05 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.491057+0000", 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020511600000000001 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.039 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.3 e360,359 2:dd5f9c08:::rbd_object_map.b3d264da6c34:head,341'257)", 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.492610+0000", 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.51424963700001, 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000210449, 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492610+0000", 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.040 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492610+0000", 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492609+0000", 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.6720000000000004e-06 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.041 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492617+0000", 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6886000000000001e-05 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492634+0000", 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.2460000000000003e-06 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.042 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492642+0000", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5142000000000005e-05 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492737+0000", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2274e-05 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492749+0000", 2023-10-09T05:01:57.043 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1123000000000003e-05 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.492820+0000", 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000210449 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.044 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.3 e360,359 2:cf283199:::rbd_header.b3eab15e15ec:head,345'301)", 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.494315+0000", 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.51254428799999, 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020752999999999999, 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494315+0000", 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.045 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494315+0000", 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494314+0000", 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8379999999999997e-06 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.046 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494324+0000", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.7210999999999998e-05 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494341+0000", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8959999999999998e-06 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494351+0000", 2023-10-09T05:01:57.047 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8574999999999998e-05 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494440+0000", 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1769e-05 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494452+0000", 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.1295000000000001e-05 2023-10-09T05:01:57.048 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.494523+0000", 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00020752999999999999 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.3 e360,359 2:cb53793b:::rbd_header.8d9ff74f3cf0:head,346'328)", 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.496041+0000", 2023-10-09T05:01:57.049 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.51081845799999, 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025326400000000001, 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496041+0000", 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.050 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496041+0000", 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496040+0000", 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2299999999999997e-06 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.051 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496049+0000", 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.2265000000000003e-05 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496092+0000", 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1268e-05 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496103+0000", 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.6225999999999997e-05 2023-10-09T05:01:57.052 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496199+0000", 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.9756000000000001e-05 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496229+0000", 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.5562999999999996e-05 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.053 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.496294+0000", 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00025326400000000001 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.3 e360,359 2:cdc59174:::rbd_data.8edd335d8504.0000000000000001:head,354'502)", 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.509749+0000", 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.49711049800001, 2023-10-09T05:01:57.054 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000218509, 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509749+0000", 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509749+0000", 2023-10-09T05:01:57.055 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509748+0000", 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.153e-05 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509759+0000", 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3785e-05 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.056 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509783+0000", 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.063e-05 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509794+0000", 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.0159000000000002e-05 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.057 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509884+0000", 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0242e-05 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509894+0000", 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.3676000000000004e-05 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.509968+0000", 2023-10-09T05:01:57.058 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000218509 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.3 360/359 [PushOp(2:c4c92e5a:::rbd_trash:head, version: 354'533, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:c4c92e5a:::rbd_trash:head@354'533, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.514819+0000", 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.49204037199999, 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00052639699999999995, 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.059 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514819+0000", 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514819+0000", 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.060 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514818+0000", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.4278e-05 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514842+0000", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.2376999999999998e-05 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514874+0000", 2023-10-09T05:01:57.061 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6086000000000002e-05 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.514891+0000", 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012784800000000001 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.515018+0000", 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2775999999999999e-05 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.062 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.515031+0000", 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00031447199999999999 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.515346+0000", 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00052639699999999995 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.063 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.5 e360,359 2:aaac2470:::rbd_header.8dab876fd1aa:head,346'132)", 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.520485+0000", 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.486375044, 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00027631299999999998, 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.064 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520485+0000", 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520485+0000", 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520483+0000", 2023-10-09T05:01:57.065 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.683e-05 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520500+0000", 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.5962999999999999e-05 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520526+0000", 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2629000000000001e-05 2023-10-09T05:01:57.066 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520539+0000", 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.9022000000000002e-05 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520638+0000", 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4399999999999999e-05 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.067 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520652+0000", 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000108912 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.520761+0000", 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00027631299999999998 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.068 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.5 e360,359 2:a7cc21f1:::rbd_id.sparse1:head,354'237)", 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.531951+0000", 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.47490874600001, 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021719600000000001, 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.069 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531951+0000", 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531951+0000", 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531950+0000", 2023-10-09T05:01:57.070 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2945999999999999e-05 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531962+0000", 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.9236999999999999e-05 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531982+0000", 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1022999999999999e-05 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.071 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.531993+0000", 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8537999999999998e-05 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.532081+0000", 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8278999999999999e-05 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.072 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.532100+0000", 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.8528000000000005e-05 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.532168+0000", 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00021719600000000001 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.073 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.5 360/359 [PushOp(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head, version: 354'240, data_included: [0~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head@354'240, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.536013+0000", 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.47084671299999, 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027897159999999998, 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.536013+0000", 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.074 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.536013+0000", 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999976 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.536011+0000", 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0014922469999999999 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.075 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.537503+0000", 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.5026e-05 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.537538+0000", 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.6186999999999999e-05 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.076 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.537554+0000", 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000155088 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.537709+0000", 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0037000000000001e-05 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.537719+0000", 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0010834530000000001 2023-10-09T05:01:57.077 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.538803+0000", 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.0027897159999999998 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.5 360/359 [PushOp(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head, version: 354'240, data_included: [1048576~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head@354'240, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2097152, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.540693+0000", 2023-10-09T05:01:57.078 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.46616688399999, 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001808811, 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.540693+0000", 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.079 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.540693+0000", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.540692+0000", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001119564 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.541811+0000", 2023-10-09T05:01:57.080 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.3090000000000001e-05 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.541834+0000", 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2984e-05 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.541847+0000", 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8830000000000001e-05 2023-10-09T05:01:57.081 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.541946+0000", 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8799999999999997e-06 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.541955+0000", 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00054671299999999999 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.082 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.542502+0000", 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001808811 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.5 360/359 [PushOp(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head, version: 354'240, data_included: [2097152~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head@354'240, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:2097152, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.544668+0000", 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.46219166399999, 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001791136, 2023-10-09T05:01:57.083 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.544668+0000", 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.544668+0000", 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.084 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.544667+0000", 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001042912 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.545710+0000", 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.8971000000000001e-05 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.085 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.545738+0000", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4195999999999999e-05 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.545753+0000", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000104527 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.545857+0000", 2023-10-09T05:01:57.086 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.2550000000000003e-06 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.545866+0000", 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00059265499999999998 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.546459+0000", 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001791136 2023-10-09T05:01:57.087 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGPush(2.5 360/359 [PushOp(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head, version: 354'240, data_included: [3145728~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:b2239686:::rbd_data.8f7912ebb78d.000000000000:head@354'240, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.548235+0000", 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.45862486499999, 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002071584, 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.088 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.548235+0000", 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.548235+0000", 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.089 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.548233+0000", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.001052007 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.549285+0000", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 3.0213999999999999e-05 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.549316+0000", 2023-10-09T05:01:57.090 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.3681e-05 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.549329+0000", 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000107284 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.549437+0000", 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.4450000000000002e-06 2023-10-09T05:01:57.091 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.549446+0000", 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00086034100000000004 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.550306+0000", 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.002071584 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.092 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.0 e360,359 2:188ae62c:::rbd_data.b3b4e5d9c632.0000000000000002:head,341'125)", 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.553062+0000", 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.45379737799999, 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000241769, 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.093 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553062+0000", 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553062+0000", 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.999999 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553061+0000", 2023-10-09T05:01:57.094 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.934e-06 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553068+0000", 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4225e-05 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553083+0000", 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.8060000000000001e-06 2023-10-09T05:01:57.095 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553091+0000", 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000117824 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553209+0000", 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0764999999999999e-05 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.096 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553220+0000", 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 8.4047000000000004e-05 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.553304+0000", 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000241769 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.097 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.0 e360,359 2:1acc5231:::rbd_data.8dab876fd1aa.0000000000000000:head,345'172)", 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.556289+0000", 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.45057016800001, 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000230552, 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556289+0000", 2023-10-09T05:01:57.098 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556289+0000", 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556288+0000", 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.8719999999999997e-06 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.099 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556298+0000", 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1285e-05 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556319+0000", 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.2585999999999999e-05 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.100 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556332+0000", 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00010744 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556439+0000", 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.4396000000000001e-05 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556454+0000", 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 6.6262999999999999e-05 2023-10-09T05:01:57.101 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.556520+0000", 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.000230552 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.0 e360,359 2:13df49cf:::rbd_object_map.8dab876fd1aa:head,346'175)", 2023-10-09T05:01:57.102 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.557964+0000", 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.44889573699999, 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024933000000000003, 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.557964+0000", 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.103 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.557964+0000", 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.557962+0000", 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0638e-05 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.557973+0000", 2023-10-09T05:01:57.104 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 2.1809000000000001e-05 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.557995+0000", 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0859999999999999e-05 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.558006+0000", 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012468000000000001 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.105 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.558130+0000", 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0519999999999999e-05 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.558141+0000", 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 7.2250999999999997e-05 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.106 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.558213+0000", 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00024933000000000003 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "description": "MOSDPGRecoveryDelete(2.0 e360,359 2:1bcf73f3:::rbd_header.b564b7b22060:head,354'206)", 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "initiated_at": "2023-10-09T04:59:48.562380+0000", 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "age": 128.44448002799999, 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00030454599999999999, 2023-10-09T05:01:57.107 INFO:teuthology.orchestra.run.smithi153.stdout: "type_data": { 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "flag_point": "started", 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "events": [ 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "initiated", 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562380+0000", 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "throttled", 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562380+0000", 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:57.108 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "header_read", 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562378+0000", 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.1449e-05 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "all_read", 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562390+0000", 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 4.8087999999999998e-05 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.109 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "dispatched", 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562438+0000", 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.8955000000000001e-05 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "queued_for_pg", 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562457+0000", 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00012181 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "reached_pg", 2023-10-09T05:01:57.110 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562578+0000", 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 1.0451e-05 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "started", 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562589+0000", 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 9.5319e-05 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: }, 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: { 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "event": "done", 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "time": "2023-10-09T04:59:48.562684+0000", 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: "duration": 0.00030454599999999999 2023-10-09T05:01:57.111 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.112 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.112 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.112 INFO:teuthology.orchestra.run.smithi153.stdout: } 2023-10-09T05:01:57.112 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2023-10-09T05:01:57.112 INFO:teuthology.orchestra.run.smithi153.stdout:} 2023-10-09T05:01:58.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:58 smithi153 bash[59565]: cluster 2023-10-09T05:01:57.029955+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:58 smithi082 bash[77063]: cluster 2023-10-09T05:01:57.029955+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:58.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:58 smithi082 bash[79733]: cluster 2023-10-09T05:01:57.029955+0000 mgr.y (mgr.34107) 1442 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:01:59.291 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:01:59.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:01:59 smithi153 bash[59565]: audit 2023-10-09T05:01:58.353505+0000 mon.a (mon.0) 2068 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:01:59.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:01:59 smithi082 bash[77063]: audit 2023-10-09T05:01:58.353505+0000 mon.a (mon.0) 2068 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:01:59.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:01:59 smithi082 bash[79733]: audit 2023-10-09T05:01:58.353505+0000 mon.a (mon.0) 2068 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:01:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:01:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:01:59] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T05:02:00.250 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: cluster 2023-10-09T05:01:59.030748+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: cluster 2023-10-09T05:01:59.282397+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: cluster 2023-10-09T05:01:59.283228+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: audit 2023-10-09T05:01:59.285202+0000 mon.a (mon.0) 2069 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: cluster 2023-10-09T05:01:59.285320+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T05:02:00.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:00 smithi153 bash[59565]: audit 2023-10-09T05:01:59.313559+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: cluster 2023-10-09T05:01:59.030748+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: cluster 2023-10-09T05:01:59.282397+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: cluster 2023-10-09T05:01:59.283228+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: audit 2023-10-09T05:01:59.285202+0000 mon.a (mon.0) 2069 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:00.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: cluster 2023-10-09T05:01:59.285320+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[77063]: audit 2023-10-09T05:01:59.313559+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: cluster 2023-10-09T05:01:59.030748+0000 mgr.y (mgr.34107) 1443 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: cluster 2023-10-09T05:01:59.282397+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: cluster 2023-10-09T05:01:59.283228+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: audit 2023-10-09T05:01:59.285202+0000 mon.a (mon.0) 2069 : audit 1 from='client.? 172.21.15.82:0/2777598165' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: cluster 2023-10-09T05:01:59.285320+0000 mon.a (mon.0) 2070 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T05:02:00.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:00 smithi082 bash[79733]: audit 2023-10-09T05:01:59.313559+0000 mon.a (mon.0) 2071 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:02.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:02 smithi153 bash[59565]: cluster 2023-10-09T05:02:01.031772+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:02.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:02 smithi082 bash[77063]: cluster 2023-10-09T05:02:01.031772+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:02.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:02 smithi082 bash[79733]: cluster 2023-10-09T05:02:01.031772+0000 mgr.y (mgr.34107) 1444 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:03.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:03 smithi153 bash[59565]: cluster 2023-10-09T05:02:03.032478+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:03.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:03 smithi082 bash[77063]: cluster 2023-10-09T05:02:03.032478+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:03.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:03 smithi082 bash[79733]: cluster 2023-10-09T05:02:03.032478+0000 mgr.y (mgr.34107) 1445 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:05.031 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:06 smithi082 bash[77063]: cluster 2023-10-09T05:02:05.033556+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:06 smithi082 bash[79733]: cluster 2023-10-09T05:02:05.033556+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:06.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:06 smithi153 bash[59565]: cluster 2023-10-09T05:02:05.033556+0000 mgr.y (mgr.34107) 1446 : cluster 0 pgmap v1633: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:08.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:08 smithi082 bash[77063]: cluster 2023-10-09T05:02:07.034412+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:08.486 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:08 smithi082 bash[79733]: cluster 2023-10-09T05:02:07.034412+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:08.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:08 smithi153 bash[59565]: cluster 2023-10-09T05:02:07.034412+0000 mgr.y (mgr.34107) 1447 : cluster 0 pgmap v1634: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:09.303 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T05:02:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:09 smithi153 bash[59565]: audit 2023-10-09T05:02:08.557275+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.82:0/1224920561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:09 smithi153 bash[59565]: audit 2023-10-09T05:02:08.558041+0000 mon.a (mon.0) 2072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:09 smithi082 bash[77063]: audit 2023-10-09T05:02:08.557275+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.82:0/1224920561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:09 smithi082 bash[77063]: audit 2023-10-09T05:02:08.558041+0000 mon.a (mon.0) 2072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:09 smithi082 bash[79733]: audit 2023-10-09T05:02:08.557275+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.82:0/1224920561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.701 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:09 smithi082 bash[79733]: audit 2023-10-09T05:02:08.558041+0000 mon.a (mon.0) 2072 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:02:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:09] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T05:02:10.257 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: cluster 2023-10-09T05:02:09.035226+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: cluster 2023-10-09T05:02:09.293044+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: audit 2023-10-09T05:02:09.294185+0000 mon.a (mon.0) 2073 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: cluster 2023-10-09T05:02:09.294258+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: cluster 2023-10-09T05:02:09.295557+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:10 smithi153 bash[59565]: cluster 2023-10-09T05:02:09.378064+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: cluster 2023-10-09T05:02:09.035226+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: cluster 2023-10-09T05:02:09.293044+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: audit 2023-10-09T05:02:09.294185+0000 mon.a (mon.0) 2073 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: cluster 2023-10-09T05:02:09.294258+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T05:02:10.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: cluster 2023-10-09T05:02:09.295557+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[77063]: cluster 2023-10-09T05:02:09.378064+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: cluster 2023-10-09T05:02:09.035226+0000 mgr.y (mgr.34107) 1448 : cluster 0 pgmap v1635: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: cluster 2023-10-09T05:02:09.293044+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: audit 2023-10-09T05:02:09.294185+0000 mon.a (mon.0) 2073 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: cluster 2023-10-09T05:02:09.294258+0000 mon.a (mon.0) 2074 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: cluster 2023-10-09T05:02:09.295557+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:10.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:10 smithi082 bash[79733]: cluster 2023-10-09T05:02:09.378064+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T05:02:12.676 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:12 smithi153 bash[59565]: cluster 2023-10-09T05:02:11.036166+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v1637: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:12 smithi082 bash[77063]: cluster 2023-10-09T05:02:11.036166+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v1637: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:12.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:12 smithi082 bash[79733]: cluster 2023-10-09T05:02:11.036166+0000 mgr.y (mgr.34107) 1449 : cluster 0 pgmap v1637: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:14.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:14 smithi153 bash[59565]: cluster 2023-10-09T05:02:13.036846+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:14.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:14 smithi082 bash[77063]: cluster 2023-10-09T05:02:13.036846+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:14.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:14 smithi082 bash[79733]: cluster 2023-10-09T05:02:13.036846+0000 mgr.y (mgr.34107) 1450 : cluster 0 pgmap v1638: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:15.026 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:15 smithi082 bash[77063]: audit 2023-10-09T05:02:14.313413+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:15 smithi082 bash[77063]: cluster 2023-10-09T05:02:15.037448+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:15.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:15 smithi082 bash[79733]: audit 2023-10-09T05:02:14.313413+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:15.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:15 smithi082 bash[79733]: cluster 2023-10-09T05:02:15.037448+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:15 smithi153 bash[59565]: audit 2023-10-09T05:02:14.313413+0000 mon.a (mon.0) 2075 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:15.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:15 smithi153 bash[59565]: cluster 2023-10-09T05:02:15.037448+0000 mgr.y (mgr.34107) 1451 : cluster 0 pgmap v1639: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:18.349 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:18 smithi082 bash[77063]: cluster 2023-10-09T05:02:17.038086+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:18.349 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:18 smithi082 bash[79733]: cluster 2023-10-09T05:02:17.038086+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:18.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:18 smithi153 bash[59565]: cluster 2023-10-09T05:02:17.038086+0000 mgr.y (mgr.34107) 1452 : cluster 0 pgmap v1640: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:19.115 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:02:19.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:19 smithi153 bash[59565]: audit 2023-10-09T05:02:18.806348+0000 mon.a (mon.0) 2076 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:19.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:19 smithi082 bash[77063]: audit 2023-10-09T05:02:18.806348+0000 mon.a (mon.0) 2076 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:19.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:19 smithi082 bash[79733]: audit 2023-10-09T05:02:18.806348+0000 mon.a (mon.0) 2076 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:19.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:19] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.33.4" 2023-10-09T05:02:20.012 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T05:02:20.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:20 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.038864+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:20 smithi153 bash[59565]: audit 2023-10-09T05:02:19.108451+0000 mon.a (mon.0) 2077 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:20 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.108557+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T05:02:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:20 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.108634+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:20.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:20 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.109186+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.038864+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[77063]: audit 2023-10-09T05:02:19.108451+0000 mon.a (mon.0) 2077 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.108557+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T05:02:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.108634+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.109186+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.038864+0000 mgr.y (mgr.34107) 1453 : cluster 0 pgmap v1641: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[79733]: audit 2023-10-09T05:02:19.108451+0000 mon.a (mon.0) 2077 : audit 1 from='client.? 172.21.15.82:0/3286121856' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.108557+0000 mon.a (mon.0) 2078 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.108634+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:20.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:20 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.109186+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:21 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.421382+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:21 smithi153 bash[59565]: cluster 2023-10-09T05:02:19.462682+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:21 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.421382+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:21 smithi082 bash[77063]: cluster 2023-10-09T05:02:19.462682+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:21 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.421382+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:21 smithi082 bash[79733]: cluster 2023-10-09T05:02:19.462682+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T05:02:22.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:22 smithi153 bash[59565]: cluster 2023-10-09T05:02:21.039821+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:22.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:22 smithi082 bash[77063]: cluster 2023-10-09T05:02:21.039821+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:22.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:22 smithi082 bash[79733]: cluster 2023-10-09T05:02:21.039821+0000 mgr.y (mgr.34107) 1454 : cluster 0 pgmap v1643: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:24.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:24 smithi153 bash[59565]: cluster 2023-10-09T05:02:23.040550+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:24.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:24 smithi082 bash[77063]: cluster 2023-10-09T05:02:23.040550+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:24.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:24 smithi082 bash[79733]: cluster 2023-10-09T05:02:23.040550+0000 mgr.y (mgr.34107) 1455 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:24.785 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:26.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:26 smithi153 bash[59565]: cluster 2023-10-09T05:02:25.041416+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v1645: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:26.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:26 smithi082 bash[77063]: cluster 2023-10-09T05:02:25.041416+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v1645: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:26.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:26 smithi082 bash[79733]: cluster 2023-10-09T05:02:25.041416+0000 mgr.y (mgr.34107) 1456 : cluster 0 pgmap v1645: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:27.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:27 smithi153 bash[59565]: cluster 2023-10-09T05:02:27.042233+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:27.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:27 smithi082 bash[77063]: cluster 2023-10-09T05:02:27.042233+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:27.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:27 smithi082 bash[79733]: cluster 2023-10-09T05:02:27.042233+0000 mgr.y (mgr.34107) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:28.583 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T05:02:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:28 smithi153 bash[59565]: audit 2023-10-09T05:02:28.513754+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.82:0/438798480' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:28.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:28 smithi153 bash[59565]: audit 2023-10-09T05:02:28.514444+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:28 smithi082 bash[77063]: audit 2023-10-09T05:02:28.513754+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.82:0/438798480' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:28.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:28 smithi082 bash[77063]: audit 2023-10-09T05:02:28.514444+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:28.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:28 smithi082 bash[79733]: audit 2023-10-09T05:02:28.513754+0000 mon.c (mon.2) 157 : audit 1 from='client.? 172.21.15.82:0/438798480' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:28.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:28 smithi082 bash[79733]: audit 2023-10-09T05:02:28.514444+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:02:29.451 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:02:29.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:29] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T05:02:29.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: cluster 2023-10-09T05:02:28.569747+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: audit 2023-10-09T05:02:28.574730+0000 mon.a (mon.0) 2080 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:29.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: cluster 2023-10-09T05:02:28.574827+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: cluster 2023-10-09T05:02:28.577837+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.042993+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: audit 2023-10-09T05:02:29.314384+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.577433+0000 mon.a (mon.0) 2083 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: cluster 2023-10-09T05:02:28.569747+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: audit 2023-10-09T05:02:28.574730+0000 mon.a (mon.0) 2080 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: cluster 2023-10-09T05:02:28.574827+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: cluster 2023-10-09T05:02:28.577837+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.042993+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:29.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: audit 2023-10-09T05:02:29.314384+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.733 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:29 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.577433+0000 mon.a (mon.0) 2083 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: cluster 2023-10-09T05:02:28.569747+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: audit 2023-10-09T05:02:28.574730+0000 mon.a (mon.0) 2080 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: cluster 2023-10-09T05:02:28.574827+0000 mon.a (mon.0) 2081 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: cluster 2023-10-09T05:02:28.577837+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.042993+0000 mgr.y (mgr.34107) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: audit 2023-10-09T05:02:29.314384+0000 mon.a (mon.0) 2082 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:29 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.577433+0000 mon.a (mon.0) 2083 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T05:02:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:30 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.424744+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:30 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.424821+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:30 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.424896+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:30 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.573969+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:30.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:30 smithi153 bash[59565]: cluster 2023-10-09T05:02:29.840188+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.424744+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.424821+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.424896+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.573969+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[77063]: cluster 2023-10-09T05:02:29.840188+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.424744+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.424821+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.424896+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.573969+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:30.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:30 smithi082 bash[79733]: cluster 2023-10-09T05:02:29.840188+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.924 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:31 smithi153 bash[59565]: cluster 2023-10-09T05:02:30.536478+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:31 smithi153 bash[59565]: cluster 2023-10-09T05:02:30.691273+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.925 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:31 smithi153 bash[59565]: cluster 2023-10-09T05:02:31.043890+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[77063]: cluster 2023-10-09T05:02:30.536478+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[77063]: cluster 2023-10-09T05:02:30.691273+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.981 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[77063]: cluster 2023-10-09T05:02:31.043890+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:31.981 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[79733]: cluster 2023-10-09T05:02:30.536478+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[79733]: cluster 2023-10-09T05:02:30.691273+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T05:02:31.982 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:31 smithi082 bash[79733]: cluster 2023-10-09T05:02:31.043890+0000 mgr.y (mgr.34107) 1459 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:34.232 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:34.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:34 smithi153 bash[59565]: cluster 2023-10-09T05:02:33.044614+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:34 smithi082 bash[77063]: cluster 2023-10-09T05:02:33.044614+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:34 smithi082 bash[79733]: cluster 2023-10-09T05:02:33.044614+0000 mgr.y (mgr.34107) 1460 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:36.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:36 smithi153 bash[59565]: cluster 2023-10-09T05:02:35.045498+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:36.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:36 smithi082 bash[77063]: cluster 2023-10-09T05:02:35.045498+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:36.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:36 smithi082 bash[79733]: cluster 2023-10-09T05:02:35.045498+0000 mgr.y (mgr.34107) 1461 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:38.303 INFO:teuthology.orchestra.run.smithi082.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:02:38.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:38 smithi153 bash[59565]: cluster 2023-10-09T05:02:37.046380+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:38 smithi153 bash[59565]: audit 2023-10-09T05:02:37.318068+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:38.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:38 smithi153 bash[59565]: audit 2023-10-09T05:02:37.898715+0000 mon.a (mon.0) 2085 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[79733]: cluster 2023-10-09T05:02:37.046380+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[79733]: audit 2023-10-09T05:02:37.318068+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[79733]: audit 2023-10-09T05:02:37.898715+0000 mon.a (mon.0) 2085 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[77063]: cluster 2023-10-09T05:02:37.046380+0000 mgr.y (mgr.34107) 1462 : cluster 0 pgmap v1653: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[77063]: audit 2023-10-09T05:02:37.318068+0000 mon.a (mon.0) 2084 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:38.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:38 smithi082 bash[77063]: audit 2023-10-09T05:02:37.898715+0000 mon.a (mon.0) 2085 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:39.188 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T05:02:39.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[77063]: cluster 2023-10-09T05:02:38.293530+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:02:39.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[77063]: audit 2023-10-09T05:02:38.294508+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:39.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[77063]: cluster 2023-10-09T05:02:38.294633+0000 mon.a (mon.0) 2087 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T05:02:39.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[77063]: cluster 2023-10-09T05:02:39.047161+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[79733]: cluster 2023-10-09T05:02:38.293530+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:02:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[79733]: audit 2023-10-09T05:02:38.294508+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[79733]: cluster 2023-10-09T05:02:38.294633+0000 mon.a (mon.0) 2087 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T05:02:39.482 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[79733]: cluster 2023-10-09T05:02:39.047161+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:39 smithi153 bash[59565]: cluster 2023-10-09T05:02:38.293530+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T05:02:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:39 smithi153 bash[59565]: audit 2023-10-09T05:02:38.294508+0000 mon.a (mon.0) 2086 : audit 1 from='client.? 172.21.15.82:0/2183172396' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:39 smithi153 bash[59565]: cluster 2023-10-09T05:02:38.294633+0000 mon.a (mon.0) 2087 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T05:02:39.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:39 smithi153 bash[59565]: cluster 2023-10-09T05:02:39.047161+0000 mgr.y (mgr.34107) 1463 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:39] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.33.4" 2023-10-09T05:02:42.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:42 smithi153 bash[59565]: cluster 2023-10-09T05:02:41.048169+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:42.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:42 smithi082 bash[77063]: cluster 2023-10-09T05:02:41.048169+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:42.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:42 smithi082 bash[79733]: cluster 2023-10-09T05:02:41.048169+0000 mgr.y (mgr.34107) 1464 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:43.743 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:44.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:44 smithi153 bash[59565]: cluster 2023-10-09T05:02:43.048864+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:44 smithi153 bash[59565]: audit 2023-10-09T05:02:43.661577+0000 mon.a (mon.0) 2088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:44 smithi153 bash[59565]: audit 2023-10-09T05:02:43.669993+0000 mon.a (mon.0) 2089 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:44 smithi153 bash[59565]: audit 2023-10-09T05:02:44.016242+0000 mon.a (mon.0) 2090 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:44 smithi153 bash[59565]: audit 2023-10-09T05:02:44.024008+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[79733]: cluster 2023-10-09T05:02:43.048864+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[79733]: audit 2023-10-09T05:02:43.661577+0000 mon.a (mon.0) 2088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[79733]: audit 2023-10-09T05:02:43.669993+0000 mon.a (mon.0) 2089 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[79733]: audit 2023-10-09T05:02:44.016242+0000 mon.a (mon.0) 2090 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[79733]: audit 2023-10-09T05:02:44.024008+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[77063]: cluster 2023-10-09T05:02:43.048864+0000 mgr.y (mgr.34107) 1465 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[77063]: audit 2023-10-09T05:02:43.661577+0000 mon.a (mon.0) 2088 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[77063]: audit 2023-10-09T05:02:43.669993+0000 mon.a (mon.0) 2089 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[77063]: audit 2023-10-09T05:02:44.016242+0000 mon.a (mon.0) 2090 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:44.732 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:44 smithi082 bash[77063]: audit 2023-10-09T05:02:44.024008+0000 mon.a (mon.0) 2091 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:45 smithi153 bash[59565]: audit 2023-10-09T05:02:44.313979+0000 mon.a (mon.0) 2092 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:45 smithi153 bash[59565]: audit 2023-10-09T05:02:44.554216+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:45 smithi153 bash[59565]: audit 2023-10-09T05:02:44.555692+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:45 smithi153 bash[59565]: audit 2023-10-09T05:02:44.567533+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:45.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:45 smithi153 bash[59565]: cluster 2023-10-09T05:02:45.049675+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[77063]: audit 2023-10-09T05:02:44.313979+0000 mon.a (mon.0) 2092 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[77063]: audit 2023-10-09T05:02:44.554216+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[77063]: audit 2023-10-09T05:02:44.555692+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[77063]: audit 2023-10-09T05:02:44.567533+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[77063]: cluster 2023-10-09T05:02:45.049675+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:45.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[79733]: audit 2023-10-09T05:02:44.313979+0000 mon.a (mon.0) 2092 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[79733]: audit 2023-10-09T05:02:44.554216+0000 mon.a (mon.0) 2093 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[79733]: audit 2023-10-09T05:02:44.555692+0000 mon.a (mon.0) 2094 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[79733]: audit 2023-10-09T05:02:44.567533+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:02:45.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:45 smithi082 bash[79733]: cluster 2023-10-09T05:02:45.049675+0000 mgr.y (mgr.34107) 1466 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:48.118 INFO:teuthology.orchestra.run.smithi082.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T05:02:48.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[77063]: cluster 2023-10-09T05:02:47.050566+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:48.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[77063]: audit 2023-10-09T05:02:47.449503+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.82:0/1041766077' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:48.365 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[77063]: audit 2023-10-09T05:02:47.450176+0000 mon.a (mon.0) 2096 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:48.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[79733]: cluster 2023-10-09T05:02:47.050566+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:48.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[79733]: audit 2023-10-09T05:02:47.449503+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.82:0/1041766077' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:48.366 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:48 smithi082 bash[79733]: audit 2023-10-09T05:02:47.450176+0000 mon.a (mon.0) 2096 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:48.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:48 smithi153 bash[59565]: cluster 2023-10-09T05:02:47.050566+0000 mgr.y (mgr.34107) 1467 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:48 smithi153 bash[59565]: audit 2023-10-09T05:02:47.449503+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.82:0/1041766077' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:48.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:48 smithi153 bash[59565]: audit 2023-10-09T05:02:47.450176+0000 mon.a (mon.0) 2096 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:02:49.067 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T05:02:49.067 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T05:02:49.067 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T05:02:49.067 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T05:02:49.067 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:02:49.068 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:02:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:49 smithi153 bash[59565]: cluster 2023-10-09T05:02:48.109065+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:49 smithi153 bash[59565]: audit 2023-10-09T05:02:48.111755+0000 mon.a (mon.0) 2097 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:49.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:49 smithi153 bash[59565]: cluster 2023-10-09T05:02:48.111869+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[77063]: cluster 2023-10-09T05:02:48.109065+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[77063]: audit 2023-10-09T05:02:48.111755+0000 mon.a (mon.0) 2097 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[77063]: cluster 2023-10-09T05:02:48.111869+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[79733]: cluster 2023-10-09T05:02:48.109065+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[79733]: audit 2023-10-09T05:02:48.111755+0000 mon.a (mon.0) 2097 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:02:49.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[79733]: cluster 2023-10-09T05:02:48.111869+0000 mon.a (mon.0) 2098 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T05:02:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:49] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.33.4" 2023-10-09T05:02:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:50 smithi153 bash[59565]: cluster 2023-10-09T05:02:49.051403+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:50 smithi082 bash[77063]: cluster 2023-10-09T05:02:49.051403+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:50 smithi082 bash[79733]: cluster 2023-10-09T05:02:49.051403+0000 mgr.y (mgr.34107) 1468 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:51 smithi153 bash[59565]: cluster 2023-10-09T05:02:50.297112+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:51 smithi153 bash[59565]: cluster 2023-10-09T05:02:50.298667+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:51 smithi153 bash[59565]: cluster 2023-10-09T05:02:50.495273+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[77063]: cluster 2023-10-09T05:02:50.297112+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[77063]: cluster 2023-10-09T05:02:50.298667+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[77063]: cluster 2023-10-09T05:02:50.495273+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[79733]: cluster 2023-10-09T05:02:50.297112+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[79733]: cluster 2023-10-09T05:02:50.298667+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:51.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:51 smithi082 bash[79733]: cluster 2023-10-09T05:02:50.495273+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:52 smithi153 bash[59565]: cluster 2023-10-09T05:02:50.495441+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:52.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:52 smithi153 bash[59565]: cluster 2023-10-09T05:02:51.052465+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:52 smithi082 bash[77063]: cluster 2023-10-09T05:02:50.495441+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:52.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:52 smithi082 bash[77063]: cluster 2023-10-09T05:02:51.052465+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:52.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:52 smithi082 bash[79733]: cluster 2023-10-09T05:02:50.495441+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T05:02:52.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:52 smithi082 bash[79733]: cluster 2023-10-09T05:02:51.052465+0000 mgr.y (mgr.34107) 1469 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:53.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:53 smithi153 bash[59565]: cluster 2023-10-09T05:02:53.053240+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:53.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:53 smithi082 bash[77063]: cluster 2023-10-09T05:02:53.053240+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:53.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:53 smithi082 bash[79733]: cluster 2023-10-09T05:02:53.053240+0000 mgr.y (mgr.34107) 1470 : cluster 0 pgmap v1663: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:53.850 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:56.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:56 smithi153 bash[59565]: cluster 2023-10-09T05:02:55.054040+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:56.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:56 smithi082 bash[77063]: cluster 2023-10-09T05:02:55.054040+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:56.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:56 smithi082 bash[79733]: cluster 2023-10-09T05:02:55.054040+0000 mgr.y (mgr.34107) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:57.419 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T05:02:57.419 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":382,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T05:02:48.104009+0000","last_up_change":"2023-10-09T04:59:46.272556+0000","last_in_change":"2023-10-09T04:56:45.302402+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"354","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":354,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":361,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6803","nonce":85623288}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6805","nonce":85623288}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6809","nonce":85623288}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6807","nonce":85623288}]},"public_addr":"172.21.15.82:6803/85623288","cluster_addr":"172.21.15.82:6805/85623288","heartbeat_back_addr":"172.21.15.82:6809/85623288","heartbeat_front_addr":"172.21.15.82:6807/85623288","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":367,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6811","nonce":1232825898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6813","nonce":1232825898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6817","nonce":1232825898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6815","nonce":1232825898}]},"public_addr":"172.21.15.82:6811/1232825898","cluster_addr":"172.21.15.82:6813/1232825898","heartbeat_back_addr":"172.21.15.82:6817/1232825898","heartbeat_front_addr":"172.21.15.82:6815/1232825898","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":217,"up_from":234,"up_thru":379,"down_at":218,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6819","nonce":2157929668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6821","nonce":2157929668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6825","nonce":2157929668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6823","nonce":2157929668}]},"public_addr":"172.21.15.82:6819/2157929668","cluster_addr":"172.21.15.82:6821/2157929668","heartbeat_back_addr":"172.21.15.82:6825/2157929668","heartbeat_front_addr":"172.21.15.82:6823/2157929668","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":370,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6827","nonce":3740620592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6829","nonce":3740620592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6833","nonce":3740620592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6831","nonce":3740620592}]},"public_addr":"172.21.15.82:6827/3740620592","cluster_addr":"172.21.15.82:6829/3740620592","heartbeat_back_addr":"172.21.15.82:6833/3740620592","heartbeat_front_addr":"172.21.15.82:6831/3740620592","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":358,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":113,"up_from":150,"up_thru":361,"down_at":114,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6817","nonce":1614088594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6819","nonce":1614088594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6823","nonce":1614088594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6821","nonce":1614088594}]},"public_addr":"172.21.15.153:6817/1614088594","cluster_addr":"172.21.15.153:6819/1614088594","heartbeat_back_addr":"172.21.15.153:6823/1614088594","heartbeat_front_addr":"172.21.15.153:6821/1614088594","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":138,"up_from":146,"up_thru":358,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6809","nonce":2741586791}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6811","nonce":2741586791}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6815","nonce":2741586791}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6813","nonce":2741586791}]},"public_addr":"172.21.15.153:6809/2741586791","cluster_addr":"172.21.15.153:6811/2741586791","heartbeat_back_addr":"172.21.15.153:6815/2741586791","heartbeat_front_addr":"172.21.15.153:6813/2741586791","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":320,"up_from":358,"up_thru":361,"down_at":321,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6825","nonce":3882514735}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6827","nonce":3882514735}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6831","nonce":3882514735}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6829","nonce":3882514735}]},"public_addr":"172.21.15.153:6825/3882514735","cluster_addr":"172.21.15.153:6827/3882514735","heartbeat_back_addr":"172.21.15.153:6831/3882514735","heartbeat_front_addr":"172.21.15.153:6829/3882514735","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":217},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":113},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":138},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":320}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/3203316964":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/2145280144":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/3596291160":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2959459015":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1415425338":"2023-10-10T04:16:27.265506+0000","172.21.15.82:6800/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/2018378994":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.153:6835/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.153:6834/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.153:0/1290610280":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/3909792950":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000","172.21.15.153:0/1754741933":"2023-10-10T04:16:27.265506+0000","172.21.15.153:0/3564317560":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/590497495":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6801/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/3169418397":"2023-10-10T04:14:52.640519+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:02:58.336 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:02:58.337 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:02:58.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:58 smithi153 bash[59565]: cluster 2023-10-09T05:02:57.054881+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:58.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:58 smithi153 bash[59565]: audit 2023-10-09T05:02:57.415059+0000 mon.a (mon.0) 2099 : audit 0 from='client.? 172.21.15.82:0/2189136643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:02:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:58 smithi082 bash[77063]: cluster 2023-10-09T05:02:57.054881+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:58.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:58 smithi082 bash[77063]: audit 2023-10-09T05:02:57.415059+0000 mon.a (mon.0) 2099 : audit 0 from='client.? 172.21.15.82:0/2189136643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:02:58.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:58 smithi082 bash[79733]: cluster 2023-10-09T05:02:57.054881+0000 mgr.y (mgr.34107) 1472 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:58.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:58 smithi082 bash[79733]: audit 2023-10-09T05:02:57.415059+0000 mon.a (mon.0) 2099 : audit 0 from='client.? 172.21.15.82:0/2189136643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:02:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:59 smithi153 bash[59565]: cluster 2023-10-09T05:02:59.055739+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:59.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:02:59 smithi153 bash[59565]: audit 2023-10-09T05:02:59.314614+0000 mon.a (mon.0) 2100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:59 smithi082 bash[77063]: cluster 2023-10-09T05:02:59.055739+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:59.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:02:59 smithi082 bash[77063]: audit 2023-10-09T05:02:59.314614+0000 mon.a (mon.0) 2100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:59 smithi082 bash[79733]: cluster 2023-10-09T05:02:59.055739+0000 mgr.y (mgr.34107) 1473 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:02:59.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:02:59 smithi082 bash[79733]: audit 2023-10-09T05:02:59.314614+0000 mon.a (mon.0) 2100 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:02:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:02:59] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T05:03:02.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:02 smithi153 bash[59565]: cluster 2023-10-09T05:03:01.056674+0000 mgr.y (mgr.34107) 1474 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:02.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:02 smithi082 bash[77063]: cluster 2023-10-09T05:03:01.056674+0000 mgr.y (mgr.34107) 1474 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:02.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:02 smithi082 bash[79733]: cluster 2023-10-09T05:03:01.056674+0000 mgr.y (mgr.34107) 1474 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:03.114 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:03.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:03 smithi082 bash[77063]: cluster 2023-10-09T05:03:03.057452+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:03.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:03 smithi082 bash[79733]: cluster 2023-10-09T05:03:03.057452+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:03.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:03 smithi153 bash[59565]: cluster 2023-10-09T05:03:03.057452+0000 mgr.y (mgr.34107) 1475 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:06.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:06 smithi082 bash[77063]: cluster 2023-10-09T05:03:05.058249+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:06.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:06 smithi082 bash[79733]: cluster 2023-10-09T05:03:05.058249+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:06.636 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T05:03:06.636 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":382,"fsid":"4ea1f4e4-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:57.352609+0000","modified":"2023-10-09T05:02:48.104009+0000","last_up_change":"2023-10-09T04:59:46.272556+0000","last_in_change":"2023-10-09T04:56:45.302402+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:05:08.936893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:09:35.235858+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"354","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":354,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:11:24.906184+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:11:25.651139+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a068a4f9-387d-4d69-afa5-d160994babe9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":361,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6803","nonce":85623288}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6805","nonce":85623288}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6809","nonce":85623288}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":85623288},{"type":"v1","addr":"172.21.15.82:6807","nonce":85623288}]},"public_addr":"172.21.15.82:6803/85623288","cluster_addr":"172.21.15.82:6805/85623288","heartbeat_back_addr":"172.21.15.82:6809/85623288","heartbeat_front_addr":"172.21.15.82:6807/85623288","state":["exists","up"]},{"osd":1,"uuid":"fcfd2467-bc5b-4cf9-8fbb-71ee2b847070","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":367,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6811","nonce":1232825898}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6813","nonce":1232825898}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6817","nonce":1232825898}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":1232825898},{"type":"v1","addr":"172.21.15.82:6815","nonce":1232825898}]},"public_addr":"172.21.15.82:6811/1232825898","cluster_addr":"172.21.15.82:6813/1232825898","heartbeat_back_addr":"172.21.15.82:6817/1232825898","heartbeat_front_addr":"172.21.15.82:6815/1232825898","state":["exists","up"]},{"osd":2,"uuid":"0b236da8-a93d-4c97-a234-b9033c7c7067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":217,"up_from":234,"up_thru":379,"down_at":218,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6819","nonce":2157929668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6821","nonce":2157929668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6825","nonce":2157929668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2157929668},{"type":"v1","addr":"172.21.15.82:6823","nonce":2157929668}]},"public_addr":"172.21.15.82:6819/2157929668","cluster_addr":"172.21.15.82:6821/2157929668","heartbeat_back_addr":"172.21.15.82:6825/2157929668","heartbeat_front_addr":"172.21.15.82:6823/2157929668","state":["exists","up"]},{"osd":3,"uuid":"0ae42ef8-7a8b-4eec-927b-01af15ae54f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":370,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6827","nonce":3740620592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6829","nonce":3740620592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6833","nonce":3740620592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":3740620592},{"type":"v1","addr":"172.21.15.82:6831","nonce":3740620592}]},"public_addr":"172.21.15.82:6827/3740620592","cluster_addr":"172.21.15.82:6829/3740620592","heartbeat_back_addr":"172.21.15.82:6833/3740620592","heartbeat_front_addr":"172.21.15.82:6831/3740620592","state":["exists","up"]},{"osd":4,"uuid":"298364f7-1ee4-436d-81d7-1ddf54d45607","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":358,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6801","nonce":2316881063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6803","nonce":2316881063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6807","nonce":2316881063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":2316881063},{"type":"v1","addr":"172.21.15.153:6805","nonce":2316881063}]},"public_addr":"172.21.15.153:6801/2316881063","cluster_addr":"172.21.15.153:6803/2316881063","heartbeat_back_addr":"172.21.15.153:6807/2316881063","heartbeat_front_addr":"172.21.15.153:6805/2316881063","state":["exists","up"]},{"osd":5,"uuid":"4d79acac-f278-473a-8dbf-9cf495c9f498","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":113,"up_from":150,"up_thru":361,"down_at":114,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6817","nonce":1614088594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6819","nonce":1614088594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6823","nonce":1614088594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":1614088594},{"type":"v1","addr":"172.21.15.153:6821","nonce":1614088594}]},"public_addr":"172.21.15.153:6817/1614088594","cluster_addr":"172.21.15.153:6819/1614088594","heartbeat_back_addr":"172.21.15.153:6823/1614088594","heartbeat_front_addr":"172.21.15.153:6821/1614088594","state":["exists","up"]},{"osd":6,"uuid":"b3727fa4-1f3c-4a8e-89b4-a9e631638f17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":138,"up_from":146,"up_thru":358,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6809","nonce":2741586791}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6811","nonce":2741586791}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6815","nonce":2741586791}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":2741586791},{"type":"v1","addr":"172.21.15.153:6813","nonce":2741586791}]},"public_addr":"172.21.15.153:6809/2741586791","cluster_addr":"172.21.15.153:6811/2741586791","heartbeat_back_addr":"172.21.15.153:6815/2741586791","heartbeat_front_addr":"172.21.15.153:6813/2741586791","state":["exists","up"]},{"osd":7,"uuid":"2ae5ef44-7804-4aa6-9343-0cd93c170de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":320,"up_from":358,"up_thru":361,"down_at":321,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6825","nonce":3882514735}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6827","nonce":3882514735}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6831","nonce":3882514735}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":3882514735},{"type":"v1","addr":"172.21.15.153:6829","nonce":3882514735}]},"public_addr":"172.21.15.153:6825/3882514735","cluster_addr":"172.21.15.153:6827/3882514735","heartbeat_back_addr":"172.21.15.153:6831/3882514735","heartbeat_front_addr":"172.21.15.153:6829/3882514735","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:42.166949+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:24.320033+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:06.501901+0000","dead_epoch":217},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.329421+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:06:28.997408+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:10.869609+0000","dead_epoch":113},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:07:52.273390+0000","dead_epoch":138},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:08:34.086169+0000","dead_epoch":320}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:0/3203316964":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:6800/2283823309":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/1174531512":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/3169418397":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/590497495":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6801/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:0/2018378994":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/2478756274":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1402928284":"2023-10-10T04:14:52.640519+0000","172.21.15.82:6800/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6801/886924328":"2023-10-10T04:01:54.491441+0000","172.21.15.82:6801/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:6800/1028221221":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1290610280":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/1663762755":"2023-10-10T04:01:54.491441+0000","172.21.15.153:6834/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/2959459015":"2023-10-10T04:17:29.105174+0000","172.21.15.153:0/1415425338":"2023-10-10T04:16:27.265506+0000","172.21.15.153:6835/4277057295":"2023-10-10T04:16:27.265506+0000","172.21.15.82:0/838245020":"2023-10-10T04:01:24.719169+0000","172.21.15.82:0/3596291160":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/3909792950":"2023-10-10T04:17:29.105174+0000","172.21.15.82:6800/3756706366":"2023-10-10T04:09:06.794971+0000","172.21.15.82:0/1703800946":"2023-10-10T04:01:54.491441+0000","172.21.15.82:0/2145280144":"2023-10-10T04:17:29.105174+0000","172.21.15.82:0/715502811":"2023-10-10T04:01:24.719169+0000","172.21.15.153:0/1754741933":"2023-10-10T04:16:27.265506+0000","172.21.15.153:0/3564317560":"2023-10-10T04:16:27.265506+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:03:06.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:06 smithi153 bash[59565]: cluster 2023-10-09T05:03:05.058249+0000 mgr.y (mgr.34107) 1476 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:07.556 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:07 smithi082 bash[79733]: audit 2023-10-09T05:03:06.632170+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.82:0/1465046223' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:03:07.556 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:07 smithi082 bash[77063]: audit 2023-10-09T05:03:06.632170+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.82:0/1465046223' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:03:07.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:03:07.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:03:07.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:03:07.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:03:07.560 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:03:07.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:03:07.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:03:07.561 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:03:07.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:07 smithi153 bash[59565]: audit 2023-10-09T05:03:06.632170+0000 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.82:0/1465046223' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:03:08.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:08 smithi153 bash[59565]: cluster 2023-10-09T05:03:07.059086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:08.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:08 smithi082 bash[77063]: cluster 2023-10-09T05:03:07.059086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:08.732 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:08 smithi082 bash[79733]: cluster 2023-10-09T05:03:07.059086+0000 mgr.y (mgr.34107) 1477 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:09.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:09 smithi153 bash[59565]: cluster 2023-10-09T05:03:09.059910+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:09.700 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:09 smithi082 bash[77063]: cluster 2023-10-09T05:03:09.059910+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:09.700 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:09 smithi082 bash[79733]: cluster 2023-10-09T05:03:09.059910+0000 mgr.y (mgr.34107) 1478 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:09] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T05:03:12.450 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.450 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.451 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.451 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.451 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.455 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.456 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.456 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:12.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:12 smithi153 bash[59565]: cluster 2023-10-09T05:03:11.060801+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:12.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:12 smithi082 bash[77063]: cluster 2023-10-09T05:03:11.060801+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:12.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:12 smithi082 bash[79733]: cluster 2023-10-09T05:03:11.060801+0000 mgr.y (mgr.34107) 1479 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:13.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:13 smithi153 bash[59565]: cluster 2023-10-09T05:03:13.061359+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:13.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:13 smithi082 bash[79733]: cluster 2023-10-09T05:03:13.061359+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:13.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:13 smithi082 bash[77063]: cluster 2023-10-09T05:03:13.061359+0000 mgr.y (mgr.34107) 1480 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:14.642 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:14 smithi082 bash[77063]: audit 2023-10-09T05:03:14.315314+0000 mon.a (mon.0) 2101 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.643 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:14 smithi082 bash[79733]: audit 2023-10-09T05:03:14.315314+0000 mon.a (mon.0) 2101 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:14 smithi153 bash[59565]: audit 2023-10-09T05:03:14.315314+0000 mon.a (mon.0) 2101 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:16.599 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:16 smithi082 bash[77063]: cluster 2023-10-09T05:03:15.062100+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:16.600 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:16 smithi082 bash[79733]: cluster 2023-10-09T05:03:15.062100+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:16.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:16 smithi153 bash[59565]: cluster 2023-10-09T05:03:15.062100+0000 mgr.y (mgr.34107) 1481 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:17.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:17 smithi153 bash[59565]: cluster 2023-10-09T05:03:17.062950+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:17.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:17 smithi082 bash[77063]: cluster 2023-10-09T05:03:17.062950+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:17.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:17 smithi082 bash[79733]: cluster 2023-10-09T05:03:17.062950+0000 mgr.y (mgr.34107) 1482 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:20.039 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:19] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:03:20.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:20 smithi082 bash[77063]: cluster 2023-10-09T05:03:19.063783+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:20.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:20 smithi082 bash[79733]: cluster 2023-10-09T05:03:19.063783+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:20.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:20 smithi153 bash[59565]: cluster 2023-10-09T05:03:19.063783+0000 mgr.y (mgr.34107) 1483 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:21.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:21 smithi082 bash[77063]: cluster 2023-10-09T05:03:21.064404+0000 mgr.y (mgr.34107) 1484 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:21.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:21 smithi082 bash[79733]: cluster 2023-10-09T05:03:21.064404+0000 mgr.y (mgr.34107) 1484 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:21.675 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:21 smithi153 bash[59565]: cluster 2023-10-09T05:03:21.064404+0000 mgr.y (mgr.34107) 1484 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:24.454 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:24 smithi153 bash[59565]: cluster 2023-10-09T05:03:23.065169+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:24.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:24 smithi082 bash[77063]: cluster 2023-10-09T05:03:23.065169+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:24.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:24 smithi082 bash[79733]: cluster 2023-10-09T05:03:23.065169+0000 mgr.y (mgr.34107) 1485 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:26.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:26 smithi153 bash[59565]: cluster 2023-10-09T05:03:25.065677+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:26.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:26 smithi082 bash[77063]: cluster 2023-10-09T05:03:25.065677+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:26.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:26 smithi082 bash[79733]: cluster 2023-10-09T05:03:25.065677+0000 mgr.y (mgr.34107) 1486 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:27.808 INFO:teuthology.orchestra.run.smithi082.stdout:627065225569 2023-10-09T05:03:27.809 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:03:28.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:28 smithi153 bash[59565]: cluster 2023-10-09T05:03:27.066514+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:28.430 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:28 smithi082 bash[79733]: cluster 2023-10-09T05:03:27.066514+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:28.431 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:28 smithi082 bash[77063]: cluster 2023-10-09T05:03:27.066514+0000 mgr.y (mgr.34107) 1487 : cluster 0 pgmap v1680: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:28.431 INFO:teuthology.orchestra.run.smithi082.stdout:442381631964 2023-10-09T05:03:28.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:03:29.862 INFO:teuthology.orchestra.run.smithi082.stdout:377957122543 2023-10-09T05:03:29.863 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:03:29.966 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:29 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:29] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:03:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:30 smithi082 bash[77063]: cluster 2023-10-09T05:03:29.067082+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:30.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:30 smithi082 bash[77063]: audit 2023-10-09T05:03:29.315693+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:30.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:30 smithi082 bash[79733]: cluster 2023-10-09T05:03:29.067082+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:30.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:30 smithi082 bash[79733]: audit 2023-10-09T05:03:29.315693+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:30.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:30 smithi153 bash[59565]: cluster 2023-10-09T05:03:29.067082+0000 mgr.y (mgr.34107) 1488 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:30.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:30 smithi153 bash[59565]: audit 2023-10-09T05:03:29.315693+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:31.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:31 smithi153 bash[59565]: cluster 2023-10-09T05:03:31.067738+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:31.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:31 smithi082 bash[77063]: cluster 2023-10-09T05:03:31.067738+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:31.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:31 smithi082 bash[79733]: cluster 2023-10-09T05:03:31.067738+0000 mgr.y (mgr.34107) 1489 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:31.999 INFO:teuthology.orchestra.run.smithi082.stdout:128849019566 2023-10-09T05:03:32.000 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:03:32.136 INFO:teuthology.orchestra.run.smithi082.stdout:644245094748 2023-10-09T05:03:32.136 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:03:32.137 INFO:teuthology.orchestra.run.smithi082.stdout:399431959018 2023-10-09T05:03:32.138 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:03:32.546 INFO:teuthology.orchestra.run.smithi082.stdout:1005022347489 2023-10-09T05:03:32.547 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:03:32.788 INFO:teuthology.orchestra.run.smithi082.stdout:1537598292014 2023-10-09T05:03:32.789 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:03:34.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:34 smithi153 bash[59565]: cluster 2023-10-09T05:03:33.068197+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:34.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:34 smithi082 bash[77063]: cluster 2023-10-09T05:03:33.068197+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:34.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:34 smithi082 bash[79733]: cluster 2023-10-09T05:03:33.068197+0000 mgr.y (mgr.34107) 1490 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:35.605 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:35.606 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:35.606 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:36.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:36 smithi153 bash[59565]: cluster 2023-10-09T05:03:35.069019+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:36.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:36 smithi082 bash[77063]: cluster 2023-10-09T05:03:35.069019+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:36.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:36 smithi082 bash[79733]: cluster 2023-10-09T05:03:35.069019+0000 mgr.y (mgr.34107) 1491 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:36.623 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:36.674 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:36.674 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:36.675 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:36.675 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:38.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:38 smithi153 bash[59565]: cluster 2023-10-09T05:03:37.069825+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:38.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:38 smithi082 bash[79733]: cluster 2023-10-09T05:03:37.069825+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:38.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:38 smithi082 bash[77063]: cluster 2023-10-09T05:03:37.069825+0000 mgr.y (mgr.34107) 1492 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:39.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:39 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:39] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:03:40.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:40 smithi153 bash[59565]: cluster 2023-10-09T05:03:39.070326+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:40.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:40 smithi082 bash[77063]: cluster 2023-10-09T05:03:39.070326+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:40.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:40 smithi082 bash[79733]: cluster 2023-10-09T05:03:39.070326+0000 mgr.y (mgr.34107) 1493 : cluster 0 pgmap v1686: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:41.597 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:41 smithi082 bash[77063]: cluster 2023-10-09T05:03:41.071380+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:41.597 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:41 smithi082 bash[79733]: cluster 2023-10-09T05:03:41.071380+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:41.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:41 smithi153 bash[59565]: cluster 2023-10-09T05:03:41.071380+0000 mgr.y (mgr.34107) 1494 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:44.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:44 smithi153 bash[59565]: cluster 2023-10-09T05:03:43.072093+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:44.464 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:44 smithi082 bash[79733]: cluster 2023-10-09T05:03:43.072093+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:44.465 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:44 smithi082 bash[77063]: cluster 2023-10-09T05:03:43.072093+0000 mgr.y (mgr.34107) 1495 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:45.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:45 smithi153 bash[59565]: audit 2023-10-09T05:03:44.315471+0000 mon.a (mon.0) 2103 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:45.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:45 smithi153 bash[59565]: audit 2023-10-09T05:03:44.645466+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:45.596 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:45 smithi082 bash[77063]: audit 2023-10-09T05:03:44.315471+0000 mon.a (mon.0) 2103 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:45.597 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:45 smithi082 bash[77063]: audit 2023-10-09T05:03:44.645466+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:45.597 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:45 smithi082 bash[79733]: audit 2023-10-09T05:03:44.315471+0000 mon.a (mon.0) 2103 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:45.597 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:45 smithi082 bash[79733]: audit 2023-10-09T05:03:44.645466+0000 mon.a (mon.0) 2104 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:46.347 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:46 smithi082 bash[77063]: cluster 2023-10-09T05:03:45.072958+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:46.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:46 smithi153 bash[59565]: cluster 2023-10-09T05:03:45.072958+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:46.730 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:46 smithi082 bash[79733]: cluster 2023-10-09T05:03:45.072958+0000 mgr.y (mgr.34107) 1496 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:48.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:48 smithi153 bash[59565]: cluster 2023-10-09T05:03:47.073555+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:48.539 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:48 smithi082 bash[77063]: cluster 2023-10-09T05:03:47.073555+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:48.540 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:48 smithi082 bash[79733]: cluster 2023-10-09T05:03:47.073555+0000 mgr.y (mgr.34107) 1497 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:49.604 INFO:teuthology.orchestra.run.smithi082.stdout:442381631968 2023-10-09T05:03:49.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:49 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:49] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T05:03:50.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:50 smithi153 bash[59565]: cluster 2023-10-09T05:03:49.074323+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:50.427 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:50 smithi153 bash[59565]: audit 2023-10-09T05:03:49.602918+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.82:0/3786437875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:03:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:50 smithi082 bash[77063]: cluster 2023-10-09T05:03:49.074323+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:50.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:50 smithi082 bash[77063]: audit 2023-10-09T05:03:49.602918+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.82:0/3786437875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:03:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:50 smithi082 bash[79733]: cluster 2023-10-09T05:03:49.074323+0000 mgr.y (mgr.34107) 1498 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:50.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:50 smithi082 bash[79733]: audit 2023-10-09T05:03:49.602918+0000 mon.c (mon.2) 160 : audit 0 from='client.? 172.21.15.82:0/3786437875' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:03:51.728 INFO:teuthology.orchestra.run.smithi082.stdout:1005022347493 2023-10-09T05:03:52.174 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:51 smithi153 bash[59565]: audit 2023-10-09T05:03:50.887987+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:51 smithi153 bash[59565]: audit 2023-10-09T05:03:50.894250+0000 mon.a (mon.0) 2106 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:51 smithi153 bash[59565]: cluster 2023-10-09T05:03:51.074916+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:52.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:51 smithi153 bash[59565]: audit 2023-10-09T05:03:51.727198+0000 mon.a (mon.0) 2107 : audit 0 from='client.? 172.21.15.82:0/1489172013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[77063]: audit 2023-10-09T05:03:50.887987+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[77063]: audit 2023-10-09T05:03:50.894250+0000 mon.a (mon.0) 2106 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[77063]: cluster 2023-10-09T05:03:51.074916+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[77063]: audit 2023-10-09T05:03:51.727198+0000 mon.a (mon.0) 2107 : audit 0 from='client.? 172.21.15.82:0/1489172013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[79733]: audit 2023-10-09T05:03:50.887987+0000 mon.a (mon.0) 2105 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.231 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[79733]: audit 2023-10-09T05:03:50.894250+0000 mon.a (mon.0) 2106 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:03:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[79733]: cluster 2023-10-09T05:03:51.074916+0000 mgr.y (mgr.34107) 1499 : cluster 0 pgmap v1692: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:52.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:51 smithi082 bash[79733]: audit 2023-10-09T05:03:51.727198+0000 mon.a (mon.0) 2107 : audit 0 from='client.? 172.21.15.82:0/1489172013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:03:53.830 INFO:teuthology.orchestra.run.smithi082.stdout:377957122549 2023-10-09T05:03:54.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631964 got 442381631968 for osd.3 2023-10-09T05:03:54.195 DEBUG:teuthology.parallel:result is None 2023-10-09T05:03:54.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:54 smithi153 bash[59565]: cluster 2023-10-09T05:03:53.075605+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:54.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:54 smithi153 bash[59565]: audit 2023-10-09T05:03:53.829788+0000 mon.a (mon.0) 2108 : audit 0 from='client.? 172.21.15.82:0/3627676542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:03:54.441 INFO:teuthology.orchestra.run.smithi082.stdout:627065225574 2023-10-09T05:03:54.480 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:54 smithi082 bash[77063]: cluster 2023-10-09T05:03:53.075605+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:54.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:54 smithi082 bash[77063]: audit 2023-10-09T05:03:53.829788+0000 mon.a (mon.0) 2108 : audit 0 from='client.? 172.21.15.82:0/3627676542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:03:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:54 smithi082 bash[79733]: cluster 2023-10-09T05:03:53.075605+0000 mgr.y (mgr.34107) 1500 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:54.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:54 smithi082 bash[79733]: audit 2023-10-09T05:03:53.829788+0000 mon.a (mon.0) 2108 : audit 0 from='client.? 172.21.15.82:0/3627676542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:03:55.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:55 smithi153 bash[59565]: audit 2023-10-09T05:03:54.440551+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.82:0/4233997822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:03:55.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:55 smithi082 bash[77063]: audit 2023-10-09T05:03:54.440551+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.82:0/4233997822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:03:55.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:55 smithi082 bash[79733]: audit 2023-10-09T05:03:54.440551+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.82:0/4233997822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:03:56.305 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1005022347489 got 1005022347493 for osd.2 2023-10-09T05:03:56.305 DEBUG:teuthology.parallel:result is None 2023-10-09T05:03:56.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:56 smithi153 bash[59565]: cluster 2023-10-09T05:03:55.076448+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:56.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:56 smithi082 bash[77063]: cluster 2023-10-09T05:03:55.076448+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:56.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:56 smithi082 bash[79733]: cluster 2023-10-09T05:03:55.076448+0000 mgr.y (mgr.34107) 1501 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:58.068 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122543 got 377957122549 for osd.0 2023-10-09T05:03:58.068 DEBUG:teuthology.parallel:result is None 2023-10-09T05:03:58.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:58 smithi153 bash[59565]: cluster 2023-10-09T05:03:57.077220+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:58.433 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:58 smithi082 bash[77063]: cluster 2023-10-09T05:03:57.077220+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:58.433 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:58 smithi082 bash[79733]: cluster 2023-10-09T05:03:57.077220+0000 mgr.y (mgr.34107) 1502 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:03:58.456 INFO:tasks.cephadm.ceph_manager.ceph:need seq 627065225569 got 627065225574 for osd.6 2023-10-09T05:03:58.456 DEBUG:teuthology.parallel:result is None 2023-10-09T05:03:58.921 INFO:teuthology.orchestra.run.smithi082.stdout:399431959024 2023-10-09T05:03:58.957 INFO:teuthology.orchestra.run.smithi082.stdout:1537598292020 2023-10-09T05:03:59.117 INFO:teuthology.orchestra.run.smithi082.stdout:644245094754 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi082.stdout:128849019572 2023-10-09T05:03:59.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:59 smithi153 bash[59565]: audit 2023-10-09T05:03:58.920841+0000 mon.a (mon.0) 2109 : audit 0 from='client.? 172.21.15.82:0/33188469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:03:59.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:59 smithi153 bash[59565]: audit 2023-10-09T05:03:58.953243+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.82:0/1000150212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:03:59.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:59 smithi153 bash[59565]: audit 2023-10-09T05:03:59.116629+0000 mon.a (mon.0) 2110 : audit 0 from='client.? 172.21.15.82:0/380599444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:03:59.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:03:59 smithi153 bash[59565]: audit 2023-10-09T05:03:59.130539+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.82:0/206556108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:03:59.479 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[77063]: audit 2023-10-09T05:03:58.920841+0000 mon.a (mon.0) 2109 : audit 0 from='client.? 172.21.15.82:0/33188469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:03:59.479 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[77063]: audit 2023-10-09T05:03:58.953243+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.82:0/1000150212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:03:59.479 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[77063]: audit 2023-10-09T05:03:59.116629+0000 mon.a (mon.0) 2110 : audit 0 from='client.? 172.21.15.82:0/380599444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:03:59.479 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[77063]: audit 2023-10-09T05:03:59.130539+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.82:0/206556108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:03:59.479 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[79733]: audit 2023-10-09T05:03:58.920841+0000 mon.a (mon.0) 2109 : audit 0 from='client.? 172.21.15.82:0/33188469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:03:59.480 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[79733]: audit 2023-10-09T05:03:58.953243+0000 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.82:0/1000150212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:03:59.480 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[79733]: audit 2023-10-09T05:03:59.116629+0000 mon.a (mon.0) 2110 : audit 0 from='client.? 172.21.15.82:0/380599444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:03:59.480 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[79733]: audit 2023-10-09T05:03:59.130539+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.82:0/206556108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:03:59.731 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:03:59 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:03:59] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T05:04:00.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:00 smithi153 bash[59565]: cluster 2023-10-09T05:03:59.077746+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:00.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:00 smithi153 bash[59565]: audit 2023-10-09T05:03:59.316588+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:00 smithi082 bash[77063]: cluster 2023-10-09T05:03:59.077746+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:00.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:00 smithi082 bash[77063]: audit 2023-10-09T05:03:59.316588+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:00 smithi082 bash[79733]: cluster 2023-10-09T05:03:59.077746+0000 mgr.y (mgr.34107) 1503 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:00.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:00 smithi082 bash[79733]: audit 2023-10-09T05:03:59.316588+0000 mon.a (mon.0) 2111 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:01.674 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:01 smithi153 bash[59565]: cluster 2023-10-09T05:04:01.078675+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:01.731 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:01 smithi082 bash[77063]: cluster 2023-10-09T05:04:01.078675+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:01.731 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:01 smithi082 bash[79733]: cluster 2023-10-09T05:04:01.078675+0000 mgr.y (mgr.34107) 1504 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:01.755 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431959018 got 399431959024 for osd.1 2023-10-09T05:04:01.755 DEBUG:teuthology.parallel:result is None 2023-10-09T05:04:01.769 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1537598292014 got 1537598292020 for osd.7 2023-10-09T05:04:01.770 DEBUG:teuthology.parallel:result is None 2023-10-09T05:04:02.080 INFO:tasks.cephadm.ceph_manager.ceph:need seq 644245094748 got 644245094754 for osd.5 2023-10-09T05:04:02.080 DEBUG:teuthology.parallel:result is None 2023-10-09T05:04:02.089 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019566 got 128849019572 for osd.4 2023-10-09T05:04:02.210 DEBUG:teuthology.parallel:result is None 2023-10-09T05:04:02.210 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:04:02.210 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:04:04.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:04 smithi153 bash[59565]: cluster 2023-10-09T05:04:03.079281+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:04.482 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:04 smithi082 bash[77063]: cluster 2023-10-09T05:04:03.079281+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:04.500 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:04 smithi082 bash[79733]: cluster 2023-10-09T05:04:03.079281+0000 mgr.y (mgr.34107) 1505 : cluster 0 pgmap v1698: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:06.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: audit 2023-10-09T05:04:04.766222+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: audit 2023-10-09T05:04:04.774178+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: cluster 2023-10-09T05:04:05.080073+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:06.175 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: audit 2023-10-09T05:04:05.241431+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:06.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: audit 2023-10-09T05:04:05.243090+0000 mon.a (mon.0) 2115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:06.176 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:05 smithi153 bash[59565]: audit 2023-10-09T05:04:05.253528+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: audit 2023-10-09T05:04:04.766222+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: audit 2023-10-09T05:04:04.774178+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: cluster 2023-10-09T05:04:05.080073+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: audit 2023-10-09T05:04:05.241431+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:06.231 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: audit 2023-10-09T05:04:05.243090+0000 mon.a (mon.0) 2115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[77063]: audit 2023-10-09T05:04:05.253528+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: audit 2023-10-09T05:04:04.766222+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: audit 2023-10-09T05:04:04.774178+0000 mon.a (mon.0) 2113 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: cluster 2023-10-09T05:04:05.080073+0000 mgr.y (mgr.34107) 1506 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: audit 2023-10-09T05:04:05.241431+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: audit 2023-10-09T05:04:05.243090+0000 mon.a (mon.0) 2115 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:04:06.232 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:05 smithi082 bash[79733]: audit 2023-10-09T05:04:05.253528+0000 mon.a (mon.0) 2116 : audit 1 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' 2023-10-09T05:04:06.641 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:08.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:08 smithi153 bash[59565]: cluster 2023-10-09T05:04:07.080920+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:08.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:08 smithi082 bash[77063]: cluster 2023-10-09T05:04:07.080920+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:08.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:08 smithi082 bash[79733]: cluster 2023-10-09T05:04:07.080920+0000 mgr.y (mgr.34107) 1507 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:09.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:09 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:04:09] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T05:04:10.270 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T05:04:10.271 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T05:04:10.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:10 smithi153 bash[59565]: cluster 2023-10-09T05:04:09.081739+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:10.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:10 smithi082 bash[77063]: cluster 2023-10-09T05:04:09.081739+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:10.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:10 smithi082 bash[79733]: cluster 2023-10-09T05:04:09.081739+0000 mgr.y (mgr.34107) 1508 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:11.230 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1701,"stamp":"2023-10-09T05:04:09.081189+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12285,"num_read_kb":25974,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":12592450,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5971332,"kb_used_data":56196,"kb_used_omap":60,"kb_used_meta":5915011,"kb_avail":743989884,"statfs":{"total":767960285184,"available":761845641216,"internally_reserved":0,"allocated":57544704,"data_stored":49396043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62376,"internal_metadata":6056971352},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004430"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487363+0000","last_change":"2023-10-09T04:56:46.364962+0000","last_active":"2023-10-09T05:02:38.487363+0000","last_peered":"2023-10-09T05:02:38.487363+0000","last_clean":"2023-10-09T05:02:38.487363+0000","last_became_active":"2023-10-09T04:56:46.363050+0000","last_became_peered":"2023-10-09T04:56:46.363050+0000","last_unstale":"2023-10-09T05:02:38.487363+0000","last_undegraded":"2023-10-09T05:02:38.487363+0000","last_fullsized":"2023-10-09T05:02:38.487363+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":307,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111352+0000","last_change":"2023-10-09T05:01:06.296446+0000","last_active":"2023-10-09T05:02:48.111352+0000","last_peered":"2023-10-09T05:02:48.111352+0000","last_clean":"2023-10-09T05:02:48.111352+0000","last_became_active":"2023-10-09T05:01:06.295944+0000","last_became_peered":"2023-10-09T05:01:06.295944+0000","last_unstale":"2023-10-09T05:02:48.111352+0000","last_undegraded":"2023-10-09T05:02:48.111352+0000","last_fullsized":"2023-10-09T05:02:48.111352+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:14.139287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711753+0000","last_change":"2023-10-09T05:00:46.394380+0000","last_active":"2023-10-09T05:02:49.711753+0000","last_peered":"2023-10-09T05:02:49.711753+0000","last_clean":"2023-10-09T05:02:49.711753+0000","last_became_active":"2023-10-09T05:00:46.392900+0000","last_became_peered":"2023-10-09T05:00:46.392900+0000","last_unstale":"2023-10-09T05:02:49.711753+0000","last_undegraded":"2023-10-09T05:02:49.711753+0000","last_fullsized":"2023-10-09T05:02:49.711753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:00.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":381,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488416+0000","last_change":"2023-10-09T04:59:47.300587+0000","last_active":"2023-10-09T05:02:38.488416+0000","last_peered":"2023-10-09T05:02:38.488416+0000","last_clean":"2023-10-09T05:02:38.488416+0000","last_became_active":"2023-10-09T04:59:47.300212+0000","last_became_peered":"2023-10-09T04:59:47.300212+0000","last_unstale":"2023-10-09T05:02:38.488416+0000","last_undegraded":"2023-10-09T05:02:38.488416+0000","last_fullsized":"2023-10-09T05:02:38.488416+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:03.609661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109872+0000","last_change":"2023-10-09T05:01:06.808235+0000","last_active":"2023-10-09T05:02:48.109872+0000","last_peered":"2023-10-09T05:02:48.109872+0000","last_clean":"2023-10-09T05:02:48.109872+0000","last_became_active":"2023-10-09T05:01:06.807871+0000","last_became_peered":"2023-10-09T05:01:06.807871+0000","last_unstale":"2023-10-09T05:02:48.109872+0000","last_undegraded":"2023-10-09T05:02:48.109872+0000","last_fullsized":"2023-10-09T05:02:48.109872+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:00.789115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":337,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488514+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T05:02:38.488514+0000","last_peered":"2023-10-09T05:02:38.488514+0000","last_clean":"2023-10-09T05:02:38.488514+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T05:02:38.488514+0000","last_undegraded":"2023-10-09T05:02:38.488514+0000","last_fullsized":"2023-10-09T05:02:38.488514+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":360,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196799+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T05:02:48.196799+0000","last_peered":"2023-10-09T05:02:48.196799+0000","last_clean":"2023-10-09T05:02:48.196799+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T05:02:48.196799+0000","last_undegraded":"2023-10-09T05:02:48.196799+0000","last_fullsized":"2023-10-09T05:02:48.196799+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":306,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.294901+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T05:02:38.294901+0000","last_peered":"2023-10-09T05:02:38.294901+0000","last_clean":"2023-10-09T05:02:38.294901+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T05:02:38.294901+0000","last_undegraded":"2023-10-09T05:02:38.294901+0000","last_fullsized":"2023-10-09T05:02:38.294901+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":338,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.309025+0000","last_change":"2023-10-09T04:59:47.299507+0000","last_active":"2023-10-09T05:02:38.309025+0000","last_peered":"2023-10-09T05:02:38.309025+0000","last_clean":"2023-10-09T05:02:38.309025+0000","last_became_active":"2023-10-09T04:59:47.299117+0000","last_became_peered":"2023-10-09T04:59:47.299117+0000","last_unstale":"2023-10-09T05:02:38.309025+0000","last_undegraded":"2023-10-09T05:02:38.309025+0000","last_fullsized":"2023-10-09T05:02:38.309025+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":162,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109893+0000","last_change":"2023-10-09T05:01:06.299381+0000","last_active":"2023-10-09T05:02:48.109893+0000","last_peered":"2023-10-09T05:02:48.109893+0000","last_clean":"2023-10-09T05:02:48.109893+0000","last_became_active":"2023-10-09T05:01:06.298866+0000","last_became_peered":"2023-10-09T05:01:06.298866+0000","last_unstale":"2023-10-09T05:02:48.109893+0000","last_undegraded":"2023-10-09T05:02:48.109893+0000","last_fullsized":"2023-10-09T05:02:48.109893+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:21.291886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":200,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489907+0000","last_change":"2023-10-09T04:59:47.304143+0000","last_active":"2023-10-09T05:02:38.489907+0000","last_peered":"2023-10-09T05:02:38.489907+0000","last_clean":"2023-10-09T05:02:38.489907+0000","last_became_active":"2023-10-09T04:59:47.303206+0000","last_became_peered":"2023-10-09T04:59:47.303206+0000","last_unstale":"2023-10-09T05:02:38.489907+0000","last_undegraded":"2023-10-09T05:02:38.489907+0000","last_fullsized":"2023-10-09T05:02:38.489907+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711753+0000","last_change":"2023-10-09T05:00:47.265174+0000","last_active":"2023-10-09T05:02:49.711753+0000","last_peered":"2023-10-09T05:02:49.711753+0000","last_clean":"2023-10-09T05:02:49.711753+0000","last_became_active":"2023-10-09T05:00:47.263977+0000","last_became_peered":"2023-10-09T05:00:47.263977+0000","last_unstale":"2023-10-09T05:02:49.711753+0000","last_undegraded":"2023-10-09T05:02:49.711753+0000","last_fullsized":"2023-10-09T05:02:49.711753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:30:48.497589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":103,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490277+0000","last_change":"2023-10-09T04:59:47.303414+0000","last_active":"2023-10-09T05:02:38.490277+0000","last_peered":"2023-10-09T05:02:38.490277+0000","last_clean":"2023-10-09T05:02:38.490277+0000","last_became_active":"2023-10-09T04:59:47.303003+0000","last_became_peered":"2023-10-09T04:59:47.303003+0000","last_unstale":"2023-10-09T05:02:38.490277+0000","last_undegraded":"2023-10-09T05:02:38.490277+0000","last_fullsized":"2023-10-09T05:02:38.490277+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490931+0000","last_change":"2023-10-09T04:56:46.360074+0000","last_active":"2023-10-09T05:02:38.490931+0000","last_peered":"2023-10-09T05:02:38.490931+0000","last_clean":"2023-10-09T05:02:38.490931+0000","last_became_active":"2023-10-09T04:56:46.359135+0000","last_became_peered":"2023-10-09T04:56:46.359135+0000","last_unstale":"2023-10-09T05:02:38.490931+0000","last_undegraded":"2023-10-09T05:02:38.490931+0000","last_fullsized":"2023-10-09T05:02:38.490931+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":215,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489302+0000","last_change":"2023-10-09T04:59:47.301222+0000","last_active":"2023-10-09T05:02:38.489302+0000","last_peered":"2023-10-09T05:02:38.489302+0000","last_clean":"2023-10-09T05:02:38.489302+0000","last_became_active":"2023-10-09T04:59:47.300692+0000","last_became_peered":"2023-10-09T04:59:47.300692+0000","last_unstale":"2023-10-09T05:02:38.489302+0000","last_undegraded":"2023-10-09T05:02:38.489302+0000","last_fullsized":"2023-10-09T05:02:38.489302+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110550+0000","last_change":"2023-10-09T05:01:06.808273+0000","last_active":"2023-10-09T05:02:48.110550+0000","last_peered":"2023-10-09T05:02:48.110550+0000","last_clean":"2023-10-09T05:02:48.110550+0000","last_became_active":"2023-10-09T05:01:06.807907+0000","last_became_peered":"2023-10-09T05:01:06.807907+0000","last_unstale":"2023-10-09T05:02:48.110550+0000","last_undegraded":"2023-10-09T05:02:48.110550+0000","last_fullsized":"2023-10-09T05:02:48.110550+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:47.665500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":378,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487207+0000","last_change":"2023-10-09T04:59:47.300887+0000","last_active":"2023-10-09T05:02:38.487207+0000","last_peered":"2023-10-09T05:02:38.487207+0000","last_clean":"2023-10-09T05:02:38.487207+0000","last_became_active":"2023-10-09T04:59:47.300489+0000","last_became_peered":"2023-10-09T04:59:47.300489+0000","last_unstale":"2023-10-09T05:02:38.487207+0000","last_undegraded":"2023-10-09T05:02:38.487207+0000","last_fullsized":"2023-10-09T05:02:38.487207+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:17.390470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":356,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.309003+0000","last_change":"2023-10-09T04:59:47.306310+0000","last_active":"2023-10-09T05:02:38.309003+0000","last_peered":"2023-10-09T05:02:38.309003+0000","last_clean":"2023-10-09T05:02:38.309003+0000","last_became_active":"2023-10-09T04:59:47.304895+0000","last_became_peered":"2023-10-09T04:59:47.304895+0000","last_unstale":"2023-10-09T05:02:38.309003+0000","last_undegraded":"2023-10-09T05:02:38.309003+0000","last_fullsized":"2023-10-09T05:02:38.309003+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488458+0000","last_change":"2023-10-09T04:59:47.305563+0000","last_active":"2023-10-09T05:02:38.488458+0000","last_peered":"2023-10-09T05:02:38.488458+0000","last_clean":"2023-10-09T05:02:38.488458+0000","last_became_active":"2023-10-09T04:59:47.304464+0000","last_became_peered":"2023-10-09T04:59:47.304464+0000","last_unstale":"2023-10-09T05:02:38.488458+0000","last_undegraded":"2023-10-09T05:02:38.488458+0000","last_fullsized":"2023-10-09T05:02:38.488458+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":378,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196732+0000","last_change":"2023-10-09T04:59:47.300755+0000","last_active":"2023-10-09T05:02:48.196732+0000","last_peered":"2023-10-09T05:02:48.196732+0000","last_clean":"2023-10-09T05:02:48.196732+0000","last_became_active":"2023-10-09T04:59:47.300264+0000","last_became_peered":"2023-10-09T04:59:47.300264+0000","last_unstale":"2023-10-09T05:02:48.196732+0000","last_undegraded":"2023-10-09T05:02:48.196732+0000","last_fullsized":"2023-10-09T05:02:48.196732+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:38.965025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":346,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488016+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T05:02:38.488016+0000","last_peered":"2023-10-09T05:02:38.488016+0000","last_clean":"2023-10-09T05:02:38.488016+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T05:02:38.488016+0000","last_undegraded":"2023-10-09T05:02:38.488016+0000","last_fullsized":"2023-10-09T05:02:38.488016+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":294,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111432+0000","last_change":"2023-10-09T04:59:47.294816+0000","last_active":"2023-10-09T05:02:48.111432+0000","last_peered":"2023-10-09T05:02:48.111432+0000","last_clean":"2023-10-09T05:02:48.111432+0000","last_became_active":"2023-10-09T04:59:47.294405+0000","last_became_peered":"2023-10-09T04:59:47.294405+0000","last_unstale":"2023-10-09T05:02:48.111432+0000","last_undegraded":"2023-10-09T05:02:48.111432+0000","last_fullsized":"2023-10-09T05:02:48.111432+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:29.691664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":194,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489965+0000","last_change":"2023-10-09T04:59:47.307447+0000","last_active":"2023-10-09T05:02:38.489965+0000","last_peered":"2023-10-09T05:02:38.489965+0000","last_clean":"2023-10-09T05:02:38.489965+0000","last_became_active":"2023-10-09T04:59:47.305940+0000","last_became_peered":"2023-10-09T04:59:47.305940+0000","last_unstale":"2023-10-09T05:02:38.489965+0000","last_undegraded":"2023-10-09T05:02:38.489965+0000","last_fullsized":"2023-10-09T05:02:38.489965+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":329,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308459+0000","last_change":"2023-10-09T04:59:47.303400+0000","last_active":"2023-10-09T05:02:38.308459+0000","last_peered":"2023-10-09T05:02:38.308459+0000","last_clean":"2023-10-09T05:02:38.308459+0000","last_became_active":"2023-10-09T04:59:47.303065+0000","last_became_peered":"2023-10-09T04:59:47.303065+0000","last_unstale":"2023-10-09T05:02:38.308459+0000","last_undegraded":"2023-10-09T05:02:38.308459+0000","last_fullsized":"2023-10-09T05:02:38.308459+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":207,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489627+0000","last_change":"2023-10-09T04:59:47.301603+0000","last_active":"2023-10-09T05:02:38.489627+0000","last_peered":"2023-10-09T05:02:38.489627+0000","last_clean":"2023-10-09T05:02:38.489627+0000","last_became_active":"2023-10-09T04:59:47.300847+0000","last_became_peered":"2023-10-09T04:59:47.300847+0000","last_unstale":"2023-10-09T05:02:38.489627+0000","last_undegraded":"2023-10-09T05:02:38.489627+0000","last_fullsized":"2023-10-09T05:02:38.489627+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.486894+0000","last_change":"2023-10-09T04:56:46.351680+0000","last_active":"2023-10-09T05:02:38.486894+0000","last_peered":"2023-10-09T05:02:38.486894+0000","last_clean":"2023-10-09T05:02:38.486894+0000","last_became_active":"2023-10-09T04:56:46.351117+0000","last_became_peered":"2023-10-09T04:56:46.351117+0000","last_unstale":"2023-10-09T05:02:38.486894+0000","last_undegraded":"2023-10-09T05:02:38.486894+0000","last_fullsized":"2023-10-09T05:02:38.486894+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":174,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489711+0000","last_change":"2023-10-09T04:59:47.302428+0000","last_active":"2023-10-09T05:02:38.489711+0000","last_peered":"2023-10-09T05:02:38.489711+0000","last_clean":"2023-10-09T05:02:38.489711+0000","last_became_active":"2023-10-09T04:59:47.300886+0000","last_became_peered":"2023-10-09T04:59:47.300886+0000","last_unstale":"2023-10-09T05:02:38.489711+0000","last_undegraded":"2023-10-09T05:02:38.489711+0000","last_fullsized":"2023-10-09T05:02:38.489711+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":381,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488119+0000","last_change":"2023-10-09T04:59:47.299651+0000","last_active":"2023-10-09T05:02:38.488119+0000","last_peered":"2023-10-09T05:02:38.488119+0000","last_clean":"2023-10-09T05:02:38.488119+0000","last_became_active":"2023-10-09T04:59:47.299314+0000","last_became_peered":"2023-10-09T04:59:47.299314+0000","last_unstale":"2023-10-09T05:02:38.488119+0000","last_undegraded":"2023-10-09T05:02:38.488119+0000","last_fullsized":"2023-10-09T05:02:38.488119+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308743+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T05:02:38.308743+0000","last_peered":"2023-10-09T05:02:38.308743+0000","last_clean":"2023-10-09T05:02:38.308743+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T05:02:38.308743+0000","last_undegraded":"2023-10-09T05:02:38.308743+0000","last_fullsized":"2023-10-09T05:02:38.308743+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":392,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:47.855465+0000","last_change":"2023-10-09T04:56:46.352313+0000","last_active":"2023-10-09T05:02:47.855465+0000","last_peered":"2023-10-09T05:02:47.855465+0000","last_clean":"2023-10-09T05:02:47.855465+0000","last_became_active":"2023-10-09T04:56:46.351505+0000","last_became_peered":"2023-10-09T04:56:46.351505+0000","last_unstale":"2023-10-09T05:02:47.855465+0000","last_undegraded":"2023-10-09T05:02:47.855465+0000","last_fullsized":"2023-10-09T05:02:47.855465+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:01.025392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":108,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110185+0000","last_change":"2023-10-09T05:01:06.296703+0000","last_active":"2023-10-09T05:02:48.110185+0000","last_peered":"2023-10-09T05:02:48.110185+0000","last_clean":"2023-10-09T05:02:48.110185+0000","last_became_active":"2023-10-09T05:01:06.296179+0000","last_became_peered":"2023-10-09T05:01:06.296179+0000","last_unstale":"2023-10-09T05:02:48.110185+0000","last_undegraded":"2023-10-09T05:02:48.110185+0000","last_fullsized":"2023-10-09T05:02:48.110185+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:40.351019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487007+0000","last_change":"2023-10-09T04:56:46.359130+0000","last_active":"2023-10-09T05:02:38.487007+0000","last_peered":"2023-10-09T05:02:38.487007+0000","last_clean":"2023-10-09T05:02:38.487007+0000","last_became_active":"2023-10-09T04:56:46.358781+0000","last_became_peered":"2023-10-09T04:56:46.358781+0000","last_unstale":"2023-10-09T05:02:38.487007+0000","last_undegraded":"2023-10-09T05:02:38.487007+0000","last_fullsized":"2023-10-09T05:02:38.487007+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":367,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487978+0000","last_change":"2023-10-09T04:56:46.360890+0000","last_active":"2023-10-09T05:02:38.487978+0000","last_peered":"2023-10-09T05:02:38.487978+0000","last_clean":"2023-10-09T05:02:38.487978+0000","last_became_active":"2023-10-09T04:56:46.360526+0000","last_became_peered":"2023-10-09T04:56:46.360526+0000","last_unstale":"2023-10-09T05:02:38.487978+0000","last_undegraded":"2023-10-09T05:02:38.487978+0000","last_fullsized":"2023-10-09T05:02:38.487978+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487319+0000","last_change":"2023-10-09T04:56:46.366317+0000","last_active":"2023-10-09T05:02:38.487319+0000","last_peered":"2023-10-09T05:02:38.487319+0000","last_clean":"2023-10-09T05:02:38.487319+0000","last_became_active":"2023-10-09T04:56:46.363159+0000","last_became_peered":"2023-10-09T04:56:46.363159+0000","last_unstale":"2023-10-09T05:02:38.487319+0000","last_undegraded":"2023-10-09T05:02:38.487319+0000","last_fullsized":"2023-10-09T05:02:38.487319+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489814+0000","last_change":"2023-10-09T04:59:47.306294+0000","last_active":"2023-10-09T05:02:38.489814+0000","last_peered":"2023-10-09T05:02:38.489814+0000","last_clean":"2023-10-09T05:02:38.489814+0000","last_became_active":"2023-10-09T04:59:47.305885+0000","last_became_peered":"2023-10-09T04:59:47.305885+0000","last_unstale":"2023-10-09T05:02:38.489814+0000","last_undegraded":"2023-10-09T05:02:38.489814+0000","last_fullsized":"2023-10-09T05:02:38.489814+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490743+0000","last_change":"2023-10-09T04:56:46.365379+0000","last_active":"2023-10-09T05:02:38.490743+0000","last_peered":"2023-10-09T05:02:38.490743+0000","last_clean":"2023-10-09T05:02:38.490743+0000","last_became_active":"2023-10-09T04:56:46.365000+0000","last_became_peered":"2023-10-09T04:56:46.365000+0000","last_unstale":"2023-10-09T05:02:38.490743+0000","last_undegraded":"2023-10-09T05:02:38.490743+0000","last_fullsized":"2023-10-09T05:02:38.490743+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109667+0000","last_change":"2023-10-09T04:59:47.303697+0000","last_active":"2023-10-09T05:02:48.109667+0000","last_peered":"2023-10-09T05:02:48.109667+0000","last_clean":"2023-10-09T05:02:48.109667+0000","last_became_active":"2023-10-09T04:59:47.303329+0000","last_became_peered":"2023-10-09T04:59:47.303329+0000","last_unstale":"2023-10-09T05:02:48.109667+0000","last_undegraded":"2023-10-09T05:02:48.109667+0000","last_fullsized":"2023-10-09T05:02:48.109667+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:38.849941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":256,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712766+0000","last_change":"2023-10-09T05:00:46.394373+0000","last_active":"2023-10-09T05:02:49.712766+0000","last_peered":"2023-10-09T05:02:49.712766+0000","last_clean":"2023-10-09T05:02:49.712766+0000","last_became_active":"2023-10-09T05:00:46.393892+0000","last_became_peered":"2023-10-09T05:00:46.393892+0000","last_unstale":"2023-10-09T05:02:49.712766+0000","last_undegraded":"2023-10-09T05:02:49.712766+0000","last_fullsized":"2023-10-09T05:02:49.712766+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:28.124258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"354'230","reported_seq":1012,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712354+0000","last_change":"2023-10-09T05:00:46.397813+0000","last_active":"2023-10-09T05:02:49.712354+0000","last_peered":"2023-10-09T05:02:49.712354+0000","last_clean":"2023-10-09T05:02:49.712354+0000","last_became_active":"2023-10-09T05:00:46.396936+0000","last_became_peered":"2023-10-09T05:00:46.396936+0000","last_unstale":"2023-10-09T05:02:49.712354+0000","last_undegraded":"2023-10-09T05:02:49.712354+0000","last_fullsized":"2023-10-09T05:02:49.712354+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":230,"log_dups_size":0,"ondisk_log_size":230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:00.513299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1269,"num_read_kb":2981,"num_write":220,"num_write_kb":1540,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308930+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T05:02:38.308930+0000","last_peered":"2023-10-09T05:02:38.308930+0000","last_clean":"2023-10-09T05:02:38.308930+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T05:02:38.308930+0000","last_undegraded":"2023-10-09T05:02:38.308930+0000","last_fullsized":"2023-10-09T05:02:38.308930+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":275,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711800+0000","last_change":"2023-10-09T05:00:46.395242+0000","last_active":"2023-10-09T05:02:49.711800+0000","last_peered":"2023-10-09T05:02:49.711800+0000","last_clean":"2023-10-09T05:02:49.711800+0000","last_became_active":"2023-10-09T05:00:46.394902+0000","last_became_peered":"2023-10-09T05:00:46.394902+0000","last_unstale":"2023-10-09T05:02:49.711800+0000","last_undegraded":"2023-10-09T05:02:49.711800+0000","last_fullsized":"2023-10-09T05:02:49.711800+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:24.054556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"354'114","reported_seq":1144,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308618+0000","last_change":"2023-10-09T04:59:14.322562+0000","last_active":"2023-10-09T05:02:38.308618+0000","last_peered":"2023-10-09T05:02:38.308618+0000","last_clean":"2023-10-09T05:02:38.308618+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T05:02:38.308618+0000","last_undegraded":"2023-10-09T05:02:38.308618+0000","last_fullsized":"2023-10-09T05:02:38.308618+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1066,"num_read_kb":3635,"num_write":105,"num_write_kb":2407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490767+0000","last_change":"2023-10-09T04:56:46.360770+0000","last_active":"2023-10-09T05:02:38.490767+0000","last_peered":"2023-10-09T05:02:38.490767+0000","last_clean":"2023-10-09T05:02:38.490767+0000","last_became_active":"2023-10-09T04:56:46.360413+0000","last_became_peered":"2023-10-09T04:56:46.360413+0000","last_unstale":"2023-10-09T05:02:38.490767+0000","last_undegraded":"2023-10-09T05:02:38.490767+0000","last_fullsized":"2023-10-09T05:02:38.490767+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711881+0000","last_change":"2023-10-09T04:57:17.362172+0000","last_active":"2023-10-09T05:02:49.711881+0000","last_peered":"2023-10-09T05:02:49.711881+0000","last_clean":"2023-10-09T05:02:49.711881+0000","last_became_active":"2023-10-09T04:57:17.361793+0000","last_became_peered":"2023-10-09T04:57:17.361793+0000","last_unstale":"2023-10-09T05:02:49.711881+0000","last_undegraded":"2023-10-09T05:02:49.711881+0000","last_fullsized":"2023-10-09T05:02:49.711881+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:18.548269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"354'346","reported_seq":1501,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.297086+0000","last_change":"2023-10-09T05:02:29.587647+0000","last_active":"2023-10-09T05:02:38.297086+0000","last_peered":"2023-10-09T05:02:38.297086+0000","last_clean":"2023-10-09T05:02:38.297086+0000","last_became_active":"2023-10-09T05:02:29.586702+0000","last_became_peered":"2023-10-09T05:02:29.586702+0000","last_unstale":"2023-10-09T05:02:38.297086+0000","last_undegraded":"2023-10-09T05:02:38.297086+0000","last_fullsized":"2023-10-09T05:02:38.297086+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":346,"log_dups_size":0,"ondisk_log_size":346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:03:34.935284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071613799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2556,"num_read_kb":3486,"num_write":298,"num_write_kb":1370,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":287,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.040666+0000","last_change":"2023-10-09T05:00:47.265150+0000","last_active":"2023-10-09T05:02:49.040666+0000","last_peered":"2023-10-09T05:02:49.040666+0000","last_clean":"2023-10-09T05:02:49.040666+0000","last_became_active":"2023-10-09T05:00:47.264137+0000","last_became_peered":"2023-10-09T05:00:47.264137+0000","last_unstale":"2023-10-09T05:02:49.040666+0000","last_undegraded":"2023-10-09T05:02:49.040666+0000","last_fullsized":"2023-10-09T05:02:49.040666+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:34.580840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":301,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712730+0000","last_change":"2023-10-09T05:00:46.395582+0000","last_active":"2023-10-09T05:02:49.712730+0000","last_peered":"2023-10-09T05:02:49.712730+0000","last_clean":"2023-10-09T05:02:49.712730+0000","last_became_active":"2023-10-09T05:00:46.395343+0000","last_became_peered":"2023-10-09T05:00:46.395343+0000","last_unstale":"2023-10-09T05:02:49.712730+0000","last_undegraded":"2023-10-09T05:02:49.712730+0000","last_fullsized":"2023-10-09T05:02:49.712730+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:50.360234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"354'533","reported_seq":2206,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308614+0000","last_change":"2023-10-09T04:59:50.323684+0000","last_active":"2023-10-09T05:02:38.308614+0000","last_peered":"2023-10-09T05:02:38.308614+0000","last_clean":"2023-10-09T05:02:38.308614+0000","last_became_active":"2023-10-09T04:59:50.320710+0000","last_became_peered":"2023-10-09T04:59:50.320710+0000","last_unstale":"2023-10-09T05:02:38.308614+0000","last_undegraded":"2023-10-09T05:02:38.308614+0000","last_fullsized":"2023-10-09T05:02:38.308614+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":533,"log_dups_size":0,"ondisk_log_size":533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2972,"num_read_kb":4653,"num_write":476,"num_write_kb":6413,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":577,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488399+0000","last_change":"2023-10-09T04:59:47.308170+0000","last_active":"2023-10-09T05:02:38.488399+0000","last_peered":"2023-10-09T05:02:38.488399+0000","last_clean":"2023-10-09T05:02:38.488399+0000","last_became_active":"2023-10-09T04:59:47.307161+0000","last_became_peered":"2023-10-09T04:59:47.307161+0000","last_unstale":"2023-10-09T05:02:38.488399+0000","last_undegraded":"2023-10-09T05:02:38.488399+0000","last_fullsized":"2023-10-09T05:02:38.488399+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110099+0000","last_change":"2023-10-09T05:01:06.298992+0000","last_active":"2023-10-09T05:02:48.110099+0000","last_peered":"2023-10-09T05:02:48.110099+0000","last_clean":"2023-10-09T05:02:48.110099+0000","last_became_active":"2023-10-09T05:01:06.298680+0000","last_became_peered":"2023-10-09T05:01:06.298680+0000","last_unstale":"2023-10-09T05:02:48.110099+0000","last_undegraded":"2023-10-09T05:02:48.110099+0000","last_fullsized":"2023-10-09T05:02:48.110099+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:57.686146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488802+0000","last_change":"2023-10-09T04:59:47.305097+0000","last_active":"2023-10-09T05:02:38.488802+0000","last_peered":"2023-10-09T05:02:38.488802+0000","last_clean":"2023-10-09T05:02:38.488802+0000","last_became_active":"2023-10-09T04:59:47.304738+0000","last_became_peered":"2023-10-09T04:59:47.304738+0000","last_unstale":"2023-10-09T05:02:38.488802+0000","last_undegraded":"2023-10-09T05:02:38.488802+0000","last_fullsized":"2023-10-09T05:02:38.488802+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"354'224","reported_seq":1176,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488951+0000","last_change":"2023-10-09T04:59:50.317479+0000","last_active":"2023-10-09T05:02:38.488951+0000","last_peered":"2023-10-09T05:02:38.488951+0000","last_clean":"2023-10-09T05:02:38.488951+0000","last_became_active":"2023-10-09T04:59:50.313715+0000","last_became_peered":"2023-10-09T04:59:50.313715+0000","last_unstale":"2023-10-09T05:02:38.488951+0000","last_undegraded":"2023-10-09T05:02:38.488951+0000","last_fullsized":"2023-10-09T05:02:38.488951+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":224,"log_dups_size":0,"ondisk_log_size":224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1839,"num_read_kb":1902,"num_write":163,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":413,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196042+0000","last_change":"2023-10-09T04:56:46.362929+0000","last_active":"2023-10-09T05:02:48.196042+0000","last_peered":"2023-10-09T05:02:48.196042+0000","last_clean":"2023-10-09T05:02:48.196042+0000","last_became_active":"2023-10-09T04:56:46.362453+0000","last_became_peered":"2023-10-09T04:56:46.362453+0000","last_unstale":"2023-10-09T05:02:48.196042+0000","last_undegraded":"2023-10-09T05:02:48.196042+0000","last_fullsized":"2023-10-09T05:02:48.196042+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:34.707239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196534+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T05:02:48.196534+0000","last_peered":"2023-10-09T05:02:48.196534+0000","last_clean":"2023-10-09T05:02:48.196534+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T05:02:48.196534+0000","last_undegraded":"2023-10-09T05:02:48.196534+0000","last_fullsized":"2023-10-09T05:02:48.196534+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"354'240","reported_seq":841,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489525+0000","last_change":"2023-10-09T04:59:50.319553+0000","last_active":"2023-10-09T05:02:38.489525+0000","last_peered":"2023-10-09T05:02:38.489525+0000","last_clean":"2023-10-09T05:02:38.489525+0000","last_became_active":"2023-10-09T04:59:50.314137+0000","last_became_peered":"2023-10-09T04:59:50.314137+0000","last_unstale":"2023-10-09T05:02:38.489525+0000","last_undegraded":"2023-10-09T05:02:38.489525+0000","last_fullsized":"2023-10-09T05:02:38.489525+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1239,"num_read_kb":4520,"num_write":217,"num_write_kb":8525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":287,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712244+0000","last_change":"2023-10-09T05:00:47.264985+0000","last_active":"2023-10-09T05:02:49.712244+0000","last_peered":"2023-10-09T05:02:49.712244+0000","last_clean":"2023-10-09T05:02:49.712244+0000","last_became_active":"2023-10-09T05:00:47.264057+0000","last_became_peered":"2023-10-09T05:00:47.264057+0000","last_unstale":"2023-10-09T05:02:49.712244+0000","last_undegraded":"2023-10-09T05:02:49.712244+0000","last_fullsized":"2023-10-09T05:02:49.712244+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:34.503675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":381,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196459+0000","last_change":"2023-10-09T04:59:47.302043+0000","last_active":"2023-10-09T05:02:48.196459+0000","last_peered":"2023-10-09T05:02:48.196459+0000","last_clean":"2023-10-09T05:02:48.196459+0000","last_became_active":"2023-10-09T04:59:47.301652+0000","last_became_peered":"2023-10-09T04:59:47.301652+0000","last_unstale":"2023-10-09T05:02:48.196459+0000","last_undegraded":"2023-10-09T05:02:48.196459+0000","last_fullsized":"2023-10-09T05:02:48.196459+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:01.559860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":108,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111491+0000","last_change":"2023-10-09T05:01:06.299766+0000","last_active":"2023-10-09T05:02:48.111491+0000","last_peered":"2023-10-09T05:02:48.111491+0000","last_clean":"2023-10-09T05:02:48.111491+0000","last_became_active":"2023-10-09T05:01:06.299339+0000","last_became_peered":"2023-10-09T05:01:06.299339+0000","last_unstale":"2023-10-09T05:02:48.111491+0000","last_undegraded":"2023-10-09T05:02:48.111491+0000","last_fullsized":"2023-10-09T05:02:48.111491+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:09.717433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"354'151","reported_seq":696,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712733+0000","last_change":"2023-10-09T05:00:46.397918+0000","last_active":"2023-10-09T05:02:49.712733+0000","last_peered":"2023-10-09T05:02:49.712733+0000","last_clean":"2023-10-09T05:02:49.712733+0000","last_became_active":"2023-10-09T05:00:46.397421+0000","last_became_peered":"2023-10-09T05:00:46.397421+0000","last_unstale":"2023-10-09T05:02:49.712733+0000","last_undegraded":"2023-10-09T05:02:49.712733+0000","last_fullsized":"2023-10-09T05:02:49.712733+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":151,"log_dups_size":0,"ondisk_log_size":151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:14.581626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":691,"num_read_kb":2271,"num_write":97,"num_write_kb":1441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":162,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109674+0000","last_change":"2023-10-09T05:01:06.299585+0000","last_active":"2023-10-09T05:02:48.109674+0000","last_peered":"2023-10-09T05:02:48.109674+0000","last_clean":"2023-10-09T05:02:48.109674+0000","last_became_active":"2023-10-09T05:01:06.299190+0000","last_became_peered":"2023-10-09T05:01:06.299190+0000","last_unstale":"2023-10-09T05:02:48.109674+0000","last_undegraded":"2023-10-09T05:02:48.109674+0000","last_fullsized":"2023-10-09T05:02:48.109674+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:06:53.232841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"354'112","reported_seq":693,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488663+0000","last_change":"2023-10-09T04:59:47.321093+0000","last_active":"2023-10-09T05:02:38.488663+0000","last_peered":"2023-10-09T05:02:38.488663+0000","last_clean":"2023-10-09T05:02:38.488663+0000","last_became_active":"2023-10-09T04:59:47.300409+0000","last_became_peered":"2023-10-09T04:59:47.300409+0000","last_unstale":"2023-10-09T05:02:38.488663+0000","last_undegraded":"2023-10-09T05:02:38.488663+0000","last_fullsized":"2023-10-09T05:02:38.488663+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":359,"num_read_kb":2026,"num_write":114,"num_write_kb":2481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196605+0000","last_change":"2023-10-09T04:56:46.422892+0000","last_active":"2023-10-09T05:02:48.196605+0000","last_peered":"2023-10-09T05:02:48.196605+0000","last_clean":"2023-10-09T05:02:48.196605+0000","last_became_active":"2023-10-09T04:56:46.363249+0000","last_became_peered":"2023-10-09T04:56:46.363249+0000","last_unstale":"2023-10-09T05:02:48.196605+0000","last_undegraded":"2023-10-09T05:02:48.196605+0000","last_fullsized":"2023-10-09T05:02:48.196605+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:48.389583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487214+0000","last_change":"2023-10-09T04:56:46.358968+0000","last_active":"2023-10-09T05:02:38.487214+0000","last_peered":"2023-10-09T05:02:38.487214+0000","last_clean":"2023-10-09T05:02:38.487214+0000","last_became_active":"2023-10-09T04:56:46.358613+0000","last_became_peered":"2023-10-09T04:56:46.358613+0000","last_unstale":"2023-10-09T05:02:38.487214+0000","last_undegraded":"2023-10-09T05:02:38.487214+0000","last_fullsized":"2023-10-09T05:02:38.487214+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":384,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487428+0000","last_change":"2023-10-09T04:59:47.301699+0000","last_active":"2023-10-09T05:02:38.487428+0000","last_peered":"2023-10-09T05:02:38.487428+0000","last_clean":"2023-10-09T05:02:38.487428+0000","last_became_active":"2023-10-09T04:59:47.301300+0000","last_became_peered":"2023-10-09T04:59:47.301300+0000","last_unstale":"2023-10-09T05:02:38.487428+0000","last_undegraded":"2023-10-09T05:02:38.487428+0000","last_fullsized":"2023-10-09T05:02:38.487428+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:57.351668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.297358+0000","last_change":"2023-10-09T05:02:29.582970+0000","last_active":"2023-10-09T05:02:38.297358+0000","last_peered":"2023-10-09T05:02:38.297358+0000","last_clean":"2023-10-09T05:02:38.297358+0000","last_became_active":"2023-10-09T05:02:29.582421+0000","last_became_peered":"2023-10-09T05:02:29.582421+0000","last_unstale":"2023-10-09T05:02:38.297358+0000","last_undegraded":"2023-10-09T05:02:38.297358+0000","last_fullsized":"2023-10-09T05:02:38.297358+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:24.256804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":333,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308935+0000","last_change":"2023-10-09T04:56:46.363704+0000","last_active":"2023-10-09T05:02:38.308935+0000","last_peered":"2023-10-09T05:02:38.308935+0000","last_clean":"2023-10-09T05:02:38.308935+0000","last_became_active":"2023-10-09T04:56:46.343047+0000","last_became_peered":"2023-10-09T04:56:46.343047+0000","last_unstale":"2023-10-09T05:02:38.308935+0000","last_undegraded":"2023-10-09T05:02:38.308935+0000","last_fullsized":"2023-10-09T05:02:38.308935+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490090+0000","last_change":"2023-10-09T04:56:46.350221+0000","last_active":"2023-10-09T05:02:38.490090+0000","last_peered":"2023-10-09T05:02:38.490090+0000","last_clean":"2023-10-09T05:02:38.490090+0000","last_became_active":"2023-10-09T04:56:46.349461+0000","last_became_peered":"2023-10-09T04:56:46.349461+0000","last_unstale":"2023-10-09T05:02:38.490090+0000","last_undegraded":"2023-10-09T05:02:38.490090+0000","last_fullsized":"2023-10-09T05:02:38.490090+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":326,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308887+0000","last_change":"2023-10-09T04:56:46.375959+0000","last_active":"2023-10-09T05:02:38.308887+0000","last_peered":"2023-10-09T05:02:38.308887+0000","last_clean":"2023-10-09T05:02:38.308887+0000","last_became_active":"2023-10-09T04:56:46.354426+0000","last_became_peered":"2023-10-09T04:56:46.354426+0000","last_unstale":"2023-10-09T05:02:38.308887+0000","last_undegraded":"2023-10-09T05:02:38.308887+0000","last_fullsized":"2023-10-09T05:02:38.308887+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.486954+0000","last_change":"2023-10-09T04:56:46.362257+0000","last_active":"2023-10-09T05:02:38.486954+0000","last_peered":"2023-10-09T05:02:38.486954+0000","last_clean":"2023-10-09T05:02:38.486954+0000","last_became_active":"2023-10-09T04:56:46.361599+0000","last_became_peered":"2023-10-09T04:56:46.361599+0000","last_unstale":"2023-10-09T05:02:38.486954+0000","last_undegraded":"2023-10-09T05:02:38.486954+0000","last_fullsized":"2023-10-09T05:02:38.486954+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":116,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111265+0000","last_change":"2023-10-09T05:01:06.298300+0000","last_active":"2023-10-09T05:02:48.111265+0000","last_peered":"2023-10-09T05:02:48.111265+0000","last_clean":"2023-10-09T05:02:48.111265+0000","last_became_active":"2023-10-09T05:01:06.297968+0000","last_became_peered":"2023-10-09T05:01:06.297968+0000","last_unstale":"2023-10-09T05:02:48.111265+0000","last_undegraded":"2023-10-09T05:02:48.111265+0000","last_fullsized":"2023-10-09T05:02:48.111265+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:19.388221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308455+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T05:02:38.308455+0000","last_peered":"2023-10-09T05:02:38.308455+0000","last_clean":"2023-10-09T05:02:38.308455+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T05:02:38.308455+0000","last_undegraded":"2023-10-09T05:02:38.308455+0000","last_fullsized":"2023-10-09T05:02:38.308455+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":382,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.197046+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T05:02:48.197046+0000","last_peered":"2023-10-09T05:02:48.197046+0000","last_clean":"2023-10-09T05:02:48.197046+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T05:02:48.197046+0000","last_undegraded":"2023-10-09T05:02:48.197046+0000","last_fullsized":"2023-10-09T05:02:48.197046+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489312+0000","last_change":"2023-10-09T04:56:46.359415+0000","last_active":"2023-10-09T05:02:38.489312+0000","last_peered":"2023-10-09T05:02:38.489312+0000","last_clean":"2023-10-09T05:02:38.489312+0000","last_became_active":"2023-10-09T04:56:46.358569+0000","last_became_peered":"2023-10-09T04:56:46.358569+0000","last_unstale":"2023-10-09T05:02:38.489312+0000","last_undegraded":"2023-10-09T05:02:38.489312+0000","last_fullsized":"2023-10-09T05:02:38.489312+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":9538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53892,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11991,"num_read_kb":25474,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8484,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":358,"seq":1537598292022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377924,"kb_used_data":12092,"kb_used_omap":13,"kb_used_meta":365810,"kb_avail":93367228,"statfs":{"total":95995035648,"available":95608041472,"internally_reserved":0,"allocated":12382208,"data_stored":11735160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13637,"internal_metadata":374590139},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.96899999999999997,"15min":1.0309999999999999},"min":{"1min":0.63,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.137,"5min":1.486,"15min":1.486},"last":1.1080000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97399999999999998,"15min":0.95999999999999996},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.212,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.121}]},{"osd":1,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.95799999999999996,"15min":0.98199999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.1519999999999999,"5min":1.444,"15min":1.444},"last":1.196},{"interface":"front","average":{"1min":0.998,"5min":0.95399999999999996,"15min":0.95799999999999996},"min":{"1min":0.65000000000000002,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.272,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.175}]},{"osd":2,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.98799999999999999,"15min":0.98699999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.242,"5min":1.482,"15min":1.482},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.002,"15min":0.99299999999999999},"min":{"1min":0.66300000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2350000000000001,"5min":1.599,"15min":1.599},"last":0.96599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0169999999999999,"15min":1.0249999999999999},"min":{"1min":0.72599999999999998,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.3220000000000001,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.048,"15min":1.0780000000000001},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.3340000000000001,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.1519999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.752,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.3169999999999999,"5min":1.734,"15min":1.734},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.98199999999999998,"15min":0.96099999999999997},"min":{"1min":0.77500000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2330000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.024}]},{"osd":5,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.028,"15min":1.0069999999999999},"min":{"1min":0.80100000000000005,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.3540000000000001,"5min":1.365,"15min":1.365},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.073,"15min":1.0569999999999999},"min":{"1min":0.70599999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2949999999999999,"5min":1.667,"15min":1.667},"last":1.0620000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.0740000000000001,"15min":1.044},"min":{"1min":0.76000000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5049999999999999,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":1.085},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.145,"15min":1.127},"min":{"1min":0.879,"5min":0.372,"15min":0.372},"max":{"1min":1.448,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.264}]}]},{"osd":6,"up_from":146,"seq":627065225578,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618236,"kb_used_data":3892,"kb_used_omap":5,"kb_used_meta":614330,"kb_avail":93126916,"statfs":{"total":95995035648,"available":95361961984,"internally_reserved":0,"allocated":3985408,"data_stored":3346425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":629074528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.877,"15min":2.1360000000000001},"min":{"1min":0.54700000000000004,"5min":0.51800000000000002,"15min":0.246},"max":{"1min":1.29,"5min":1.5509999999999999,"15min":81.287999999999997},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.876,"5min":0.91900000000000004,"15min":1.8979999999999999},"min":{"1min":0.621,"5min":0.40600000000000003,"15min":0.254},"max":{"1min":1.151,"5min":1.5369999999999999,"15min":81.168999999999997},"last":1.0569999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91100000000000003,"15min":1.6399999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.28799999999999998},"max":{"1min":1.2270000000000001,"5min":1.2929999999999999,"15min":40.709000000000003},"last":1.21},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.91100000000000003,"15min":1.7230000000000001},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.28299999999999997},"max":{"1min":1.194,"5min":1.5269999999999999,"15min":40.414000000000001},"last":0.88}]},{"osd":2,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.92900000000000005,"15min":1.919},"min":{"1min":0.72499999999999998,"5min":0.59899999999999998,"15min":0.31900000000000001},"max":{"1min":1.258,"5min":1.464,"15min":97.284999999999997},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.94699999999999995,"15min":1.732},"min":{"1min":0.57599999999999996,"5min":0.55600000000000005,"15min":0.30599999999999999},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":97.069000000000003},"last":0.71599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97799999999999998,"15min":1.423},"min":{"1min":0.65600000000000003,"5min":0.54400000000000004,"15min":0.318},"max":{"1min":1.357,"5min":1.357,"15min":12.050000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0089999999999999,"15min":1.52},"min":{"1min":0.67400000000000004,"5min":0.496,"15min":0.34499999999999997},"max":{"1min":1.3440000000000001,"5min":1.54,"15min":18.387},"last":1.135}]},{"osd":4,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.97199999999999998,"15min":0.88900000000000001},"min":{"1min":0.64400000000000002,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":1.3640000000000001,"5min":1.5960000000000001,"15min":4.9139999999999997},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0429999999999999,"15min":1.216},"min":{"1min":0.59899999999999998,"5min":0.44900000000000001,"15min":0.34499999999999997},"max":{"1min":1.359,"5min":1.6459999999999999,"15min":10.428000000000001},"last":0.99099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:03:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.046,"15min":1.2030000000000001},"min":{"1min":0.66600000000000004,"5min":0.38200000000000001,"15min":0.28000000000000003},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":29.077999999999999},"last":1.242},{"interface":"front","average":{"1min":1.181,"5min":1.0740000000000001,"15min":1.645},"min":{"1min":0.77900000000000003,"5min":0.41499999999999998,"15min":0.34999999999999998},"max":{"1min":1.4379999999999999,"5min":1.738,"15min":28.742000000000001},"last":1.1160000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.083,"15min":1.0349999999999999},"min":{"1min":0.75600000000000001,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.4039999999999999,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.026},{"interface":"front","average":{"1min":1.167,"5min":1.1659999999999999,"15min":1.1279999999999999},"min":{"1min":0.89300000000000002,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.524,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.173}]}]},{"osd":1,"up_from":93,"seq":399431959027,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917464,"kb_used_data":4112,"kb_used_omap":6,"kb_used_meta":913337,"kb_avail":92827688,"statfs":{"total":95995035648,"available":95055552512,"internally_reserved":0,"allocated":4210688,"data_stored":2852572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":935257100},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:06 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.67700000000000005,"15min":1.536},"min":{"1min":0.437,"5min":0.28399999999999997,"15min":0.183},"max":{"1min":1.1379999999999999,"5min":1.25,"15min":47.274000000000001},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.77000000000000002,"15min":1.363},"min":{"1min":0.54200000000000004,"5min":0.27100000000000002,"15min":0.23400000000000001},"max":{"1min":1.1539999999999999,"5min":1.363,"15min":47.372999999999998},"last":1.143}]},{"osd":2,"last update":"Mon Oct 9 05:03:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.74399999999999999,"15min":1.1879999999999999},"min":{"1min":0.42599999999999999,"5min":0.34300000000000003,"15min":0.21299999999999999},"max":{"1min":1.784,"5min":1.925,"15min":10.004},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.70499999999999996,"15min":1.3169999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.20999999999999999},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":15.048},"last":0.80700000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.71599999999999997,"15min":1.77},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.23300000000000001},"max":{"1min":1.0840000000000001,"5min":1.177,"15min":68.974000000000004},"last":1.03},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.84899999999999998,"15min":1.8200000000000001},"min":{"1min":0.45000000000000001,"5min":0.38400000000000001,"15min":0.26000000000000001},"max":{"1min":1.266,"5min":1.3049999999999999,"15min":69.183000000000007},"last":1.2450000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:03:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.86099999999999999,"15min":1.411},"min":{"1min":0.59299999999999997,"5min":0.36799999999999999,"15min":0.308},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":10.398},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.94599999999999995,"15min":1.3600000000000001},"min":{"1min":0.76800000000000002,"5min":0.505,"15min":0.34499999999999997},"max":{"1min":1.78,"5min":1.78,"15min":9.827},"last":1.077}]},{"osd":5,"last update":"Mon Oct 9 05:03:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.012,"15min":1.4199999999999999},"min":{"1min":0.70599999999999996,"5min":0.54300000000000004,"15min":0.32500000000000001},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":10.438000000000001},"last":1.4119999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.96199999999999997,"15min":1.5660000000000001},"min":{"1min":0.66900000000000004,"5min":0.55100000000000005,"15min":0.33100000000000002},"max":{"1min":1.3009999999999999,"5min":1.448,"15min":10.435},"last":1.1850000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:03:06 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.0760000000000001,"15min":1.4910000000000001},"min":{"1min":0.80900000000000005,"5min":0.52300000000000002,"15min":0.35099999999999998},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":18.295000000000002},"last":1.506},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.1559999999999999,"15min":1.752},"min":{"1min":0.84199999999999997,"5min":0.54900000000000004,"15min":0.41999999999999998},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":68.581000000000003},"last":1.5760000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:03:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.1100000000000001,"15min":1.119},"min":{"1min":0.63800000000000001,"5min":0.624,"15min":0.624},"max":{"1min":1.407,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.3300000000000001},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.1779999999999999,"15min":1.1759999999999999},"min":{"1min":0.81399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.5589999999999999,"5min":1.71,"15min":1.71},"last":1.653}]}]},{"osd":0,"up_from":88,"seq":377957122552,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946460,"kb_used_data":8852,"kb_used_omap":9,"kb_used_meta":937590,"kb_avail":92798692,"statfs":{"total":95995035648,"available":95025860608,"internally_reserved":0,"allocated":9064448,"data_stored":7650535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":960092657},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67300000000000004,"15min":1.4690000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.18099999999999999},"max":{"1min":1.054,"5min":1.2150000000000001,"15min":62.396000000000001},"last":0.441},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.70399999999999996,"15min":1.571},"min":{"1min":0.39900000000000002,"5min":0.376,"15min":0.19400000000000001},"max":{"1min":1.0649999999999999,"5min":1.22,"15min":62.496000000000002},"last":0.64200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.72699999999999998,"15min":1.113},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.23699999999999999},"max":{"1min":0.94499999999999995,"5min":2.8610000000000002,"15min":13.493},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.76200000000000001,"15min":1.3240000000000001},"min":{"1min":0.40400000000000003,"5min":0.38600000000000001,"15min":0.246},"max":{"1min":1.2270000000000001,"5min":1.2929999999999999,"15min":10.500999999999999},"last":0.74399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80500000000000005,"15min":1.349},"min":{"1min":0.377,"5min":0.377,"15min":0.192},"max":{"1min":1.333,"5min":1.4370000000000001,"15min":13.545999999999999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.871,"5min":0.876,"15min":1.5760000000000001},"min":{"1min":0.54800000000000004,"5min":0.45200000000000001,"15min":0.21199999999999999},"max":{"1min":1.294,"5min":1.635,"15min":13.544},"last":0.84799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.92400000000000004,"15min":1.5389999999999999},"min":{"1min":0.66500000000000004,"5min":0.56499999999999995,"15min":0.33300000000000002},"max":{"1min":1.157,"5min":1.371,"15min":14.996},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0609999999999999,"15min":1.3180000000000001},"min":{"1min":0.79600000000000004,"5min":0.66300000000000003,"15min":0.32000000000000001},"max":{"1min":1.385,"5min":1.468,"15min":13.417},"last":0.88800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.05,"15min":1.5089999999999999},"min":{"1min":0.77500000000000002,"5min":0.61299999999999999,"15min":0.33800000000000002},"max":{"1min":1.464,"5min":1.5780000000000001,"15min":10.621},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.046,"15min":1.4870000000000001},"min":{"1min":0.68000000000000005,"5min":0.61799999999999999,"15min":0.32600000000000001},"max":{"1min":1.3899999999999999,"5min":1.5409999999999999,"15min":13.374000000000001},"last":0.82999999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.0980000000000001,"15min":1.448},"min":{"1min":0.74299999999999999,"5min":0.58599999999999997,"15min":0.33500000000000002},"max":{"1min":1.4099999999999999,"5min":1.581,"15min":13.445},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.046,"5min":1.1180000000000001,"15min":1.6439999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.38700000000000001},"max":{"1min":1.595,"5min":1.6970000000000001,"15min":13.477},"last":1.0229999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.1399999999999999,"15min":1.1859999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.4219999999999999,"5min":1.647,"15min":1.647},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0509999999999999,"15min":1.099},"min":{"1min":0.70499999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.2749999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.96399999999999997}]}]},{"osd":2,"up_from":234,"seq":1005022347497,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328008,"kb_used_data":8256,"kb_used_omap":13,"kb_used_meta":319730,"kb_avail":93417144,"statfs":{"total":95995035648,"available":95659155456,"internally_reserved":0,"allocated":8454144,"data_stored":7051275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":327404415},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.65500000000000003,"15min":3.0920000000000001},"min":{"1min":0.35799999999999998,"5min":0.23799999999999999,"15min":0.186},"max":{"1min":1.4370000000000001,"5min":1.4370000000000001,"15min":283.30799999999999},"last":0.754},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.71099999999999997,"15min":3.4119999999999999},"min":{"1min":0.41999999999999998,"5min":0.33800000000000002,"15min":0.17999999999999999},"max":{"1min":1.8560000000000001,"5min":1.8560000000000001,"15min":282.24099999999999},"last":0.63900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.76500000000000001,"15min":4.1239999999999997},"min":{"1min":0.51200000000000001,"5min":0.32700000000000001,"15min":0.218},"max":{"1min":1.8149999999999999,"5min":1.8149999999999999,"15min":267.54000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.746,"15min":4.2039999999999997},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.22900000000000001},"max":{"1min":1.833,"5min":1.833,"15min":268.12200000000001},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.77300000000000002,"15min":1.371},"min":{"1min":0.60899999999999999,"5min":0.309,"15min":0.23100000000000001},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":14.334},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.751,"15min":1.331},"min":{"1min":0.35399999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":1.2569999999999999,"5min":1.327,"15min":13.167999999999999},"last":0.67100000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.95199999999999996,"15min":1.254},"min":{"1min":0.63600000000000001,"5min":0.59599999999999997,"15min":0.33000000000000002},"max":{"1min":1.655,"5min":1.655,"15min":10.051},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.99299999999999999,"15min":1.2490000000000001},"min":{"1min":0.57499999999999996,"5min":0.54900000000000004,"15min":0.316},"max":{"1min":1.7549999999999999,"5min":1.7549999999999999,"15min":14.063000000000001},"last":1.0269999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.008,"15min":1.266},"min":{"1min":0.76400000000000001,"5min":0.56299999999999994,"15min":0.32800000000000001},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":10.856999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.071,"5min":1.014,"15min":1.3999999999999999},"min":{"1min":0.73699999999999999,"5min":0.50600000000000001,"15min":0.316},"max":{"1min":1.79,"5min":1.79,"15min":11.75},"last":0.90900000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.046,"15min":1.4630000000000001},"min":{"1min":0.70099999999999996,"5min":0.51600000000000001,"15min":0.34999999999999998},"max":{"1min":1.677,"5min":1.677,"15min":14.012},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.069,"15min":1.4279999999999999},"min":{"1min":0.82999999999999996,"5min":0.60299999999999998,"15min":0.34699999999999998},"max":{"1min":1.6679999999999999,"5min":1.679,"15min":10.890000000000001},"last":0.92900000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:03:54 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.115,"15min":1.1220000000000001},"min":{"1min":0.73299999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.355,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.149,"5min":1.157,"15min":1.1519999999999999},"min":{"1min":0.70399999999999996,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":3.915,"5min":3.915,"15min":3.915},"last":0.96199999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631972,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917984,"kb_used_data":4184,"kb_used_omap":5,"kb_used_meta":913786,"kb_avail":92827168,"statfs":{"total":95995035648,"available":95055020032,"internally_reserved":0,"allocated":4284416,"data_stored":2863792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":935717038},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.67700000000000005,"15min":3.6699999999999999},"min":{"1min":0.46700000000000003,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":1.1100000000000001,"5min":1.236,"15min":263.22199999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.72799999999999998,"15min":3.9340000000000002},"min":{"1min":0.498,"5min":0.32600000000000001,"15min":0.23200000000000001},"max":{"1min":1.089,"5min":1.4259999999999999,"15min":261.74599999999998},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.66100000000000003,"15min":2.101},"min":{"1min":0.42999999999999999,"5min":0.27300000000000002,"15min":0.21199999999999999},"max":{"1min":0.97499999999999998,"5min":1.141,"15min":143.34100000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.70399999999999996,"15min":2.3370000000000002},"min":{"1min":0.41099999999999998,"5min":0.29199999999999998,"15min":0.20399999999999999},"max":{"1min":1.137,"5min":1.202,"15min":149.15700000000001},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.74199999999999999,"15min":1.202},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.25},"max":{"1min":1.0509999999999999,"5min":1.4410000000000001,"15min":16.620000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.73399999999999999,"15min":1.613},"min":{"1min":0.44,"5min":0.33900000000000002,"15min":0.20899999999999999},"max":{"1min":1.1319999999999999,"5min":1.3089999999999999,"15min":16.649000000000001},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.93000000000000005,"15min":1.226},"min":{"1min":0.65200000000000002,"5min":0.499,"15min":0.32100000000000001},"max":{"1min":1.258,"5min":1.3759999999999999,"15min":8.9930000000000003},"last":0.76100000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.97299999999999998,"15min":1.4139999999999999},"min":{"1min":0.64300000000000002,"5min":0.58899999999999997,"15min":0.32000000000000001},"max":{"1min":1.304,"5min":1.546,"15min":15.522},"last":0.88900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.99199999999999999,"15min":1.8080000000000001},"min":{"1min":0.65500000000000003,"5min":0.56000000000000005,"15min":0.34799999999999998},"max":{"1min":1.28,"5min":1.825,"15min":23.073},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.024,"15min":1.7290000000000001},"min":{"1min":0.72799999999999998,"5min":0.61099999999999999,"15min":0.379},"max":{"1min":1.3620000000000001,"5min":1.673,"15min":22.832000000000001},"last":0.91900000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98799999999999999,"15min":1.409},"min":{"1min":0.67000000000000004,"5min":0.52500000000000002,"15min":0.32700000000000001},"max":{"1min":1.468,"5min":1.712,"15min":10.102},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0289999999999999,"15min":1.3440000000000001},"min":{"1min":0.78700000000000003,"5min":0.49399999999999999,"15min":0.35999999999999999},"max":{"1min":1.3979999999999999,"5min":1.631,"15min":8.907},"last":0.92500000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:03:58 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.1180000000000001,"15min":1.1519999999999999},"min":{"1min":0.78200000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.222,"5min":1.823,"15min":1.823},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1319999999999999,"15min":1.1699999999999999},"min":{"1min":0.69699999999999995,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.2370000000000001,"5min":1.772,"15min":1.772},"last":0.84599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019574,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902012,"kb_used_data":7412,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":92843140,"statfs":{"total":95995035648,"available":95071375360,"internally_reserved":0,"allocated":7589888,"data_stored":6950418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.84899999999999998,"15min":1.4199999999999999},"min":{"1min":0.59399999999999997,"5min":0.48999999999999999,"15min":0.26400000000000001},"max":{"1min":1.23,"5min":1.23,"15min":51.948999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.89000000000000001,"15min":1.341},"min":{"1min":0.62,"5min":0.46200000000000002,"15min":0.26300000000000001},"max":{"1min":1.1659999999999999,"5min":1.603,"15min":52.222000000000001},"last":0.67700000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.88600000000000001,"15min":1.9830000000000001},"min":{"1min":0.628,"5min":0.47199999999999998,"15min":0.27700000000000002},"max":{"1min":1.409,"5min":1.409,"15min":81.114999999999995},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.94599999999999995,"15min":2.0579999999999998},"min":{"1min":0.71399999999999997,"5min":0.53100000000000003,"15min":0.27600000000000002},"max":{"1min":1.456,"5min":1.456,"15min":80.819999999999993},"last":0.92900000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.88900000000000001,"15min":1.0680000000000001},"min":{"1min":0.55000000000000004,"5min":0.41899999999999998,"15min":0.29799999999999999},"max":{"1min":1.254,"5min":1.3089999999999999,"15min":8.6310000000000002},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.079,"5min":0.99099999999999999,"15min":1.242},"min":{"1min":0.67200000000000004,"5min":0.54600000000000004,"15min":0.28599999999999998},"max":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":15.34},"last":1.0509999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.94699999999999995,"15min":1.258},"min":{"1min":0.78800000000000003,"5min":0.57499999999999996,"15min":0.29699999999999999},"max":{"1min":1.2889999999999999,"5min":1.5149999999999999,"15min":32.362000000000002},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.008,"15min":1.4770000000000001},"min":{"1min":0.82399999999999995,"5min":0.47999999999999998,"15min":0.311},"max":{"1min":1.365,"5min":1.365,"15min":32.387},"last":0.79400000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.98199999999999998,"15min":1.5640000000000001},"min":{"1min":0.63300000000000001,"5min":0.46500000000000002,"15min":0.311},"max":{"1min":1.47,"5min":1.47,"15min":78.885000000000005},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.93700000000000006,"15min":1.3500000000000001},"min":{"1min":0.73599999999999999,"5min":0.40000000000000002,"15min":0.307},"max":{"1min":1.2190000000000001,"5min":1.349,"15min":78.472999999999999},"last":0.96399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.99299999999999999,"15min":0.999},"min":{"1min":0.81200000000000006,"5min":0.53500000000000003,"15min":0.316},"max":{"1min":1.337,"5min":1.452,"15min":8.6999999999999993},"last":1.2929999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.0369999999999999,"15min":1.3580000000000001},"min":{"1min":0.77800000000000002,"5min":0.502,"15min":0.30399999999999999},"max":{"1min":1.365,"5min":1.5609999999999999,"15min":9.3759999999999994},"last":1.1850000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.107,"15min":1.038},"min":{"1min":0.82499999999999996,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.3720000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.028},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.149,"15min":1.075},"min":{"1min":0.82499999999999996,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.7110000000000001,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":1.204}]}]},{"osd":5,"up_from":150,"seq":644245094756,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963244,"kb_used_data":7396,"kb_used_omap":6,"kb_used_meta":955833,"kb_avail":92781908,"statfs":{"total":95995035648,"available":95008673792,"internally_reserved":0,"allocated":7573504,"data_stored":6945866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":978773267},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.92400000000000004,"15min":3.7999999999999998},"min":{"1min":0.623,"5min":0.374,"15min":0.26500000000000001},"max":{"1min":1.6850000000000001,"5min":2.3399999999999999,"15min":363.64499999999998},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.92900000000000005,"15min":3.8929999999999998},"min":{"1min":0.53200000000000003,"5min":0.50700000000000001,"15min":0.26600000000000001},"max":{"1min":1.3999999999999999,"5min":1.452,"15min":363.26900000000001},"last":0.79800000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.88,"15min":2.6179999999999999},"min":{"1min":0.621,"5min":0.47399999999999998,"15min":0.308},"max":{"1min":1.333,"5min":1.3959999999999999,"15min":130.762},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.159,"5min":0.97199999999999998,"15min":2.847},"min":{"1min":0.69299999999999995,"5min":0.53300000000000003,"15min":0.29499999999999998},"max":{"1min":4.6559999999999997,"5min":4.6559999999999997,"15min":130.86199999999999},"last":0.98799999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.93000000000000005,"15min":1.712},"min":{"1min":0.624,"5min":0.52100000000000002,"15min":0.28399999999999997},"max":{"1min":1.5860000000000001,"5min":1.5860000000000001,"15min":208.40100000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.94399999999999995,"15min":1.9490000000000001},"min":{"1min":0.73799999999999999,"5min":0.55000000000000004,"15min":0.29899999999999999},"max":{"1min":1.1919999999999999,"5min":1.4890000000000001,"15min":208.078},"last":1.2290000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.006,"15min":2.153},"min":{"1min":0.72499999999999998,"5min":0.625,"15min":0.31900000000000001},"max":{"1min":4.7729999999999997,"5min":4.7729999999999997,"15min":130.69200000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.216,"5min":1.0680000000000001,"15min":2.2949999999999999},"min":{"1min":0.64300000000000002,"5min":0.624,"15min":0.33000000000000002},"max":{"1min":4.734,"5min":4.734,"15min":130.53399999999999},"last":1.151}]},{"osd":4,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.048,"15min":1.208},"min":{"1min":0.74199999999999999,"5min":0.44500000000000001,"15min":0.34300000000000003},"max":{"1min":4.8129999999999997,"5min":4.8129999999999997,"15min":12.734},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.056,"15min":1.135},"min":{"1min":0.68000000000000005,"5min":0.42299999999999999,"15min":0.28100000000000003},"max":{"1min":4.8010000000000002,"5min":4.8010000000000002,"15min":8.3409999999999993},"last":1.1319999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":1.103,"15min":1.0900000000000001},"min":{"1min":0.76900000000000002,"5min":0.377,"15min":0.33900000000000002},"max":{"1min":4.7599999999999998,"5min":4.7599999999999998,"15min":7.2489999999999997},"last":1.2569999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.1279999999999999,"15min":1.3500000000000001},"min":{"1min":0.79600000000000004,"5min":0.44800000000000001,"15min":0.28499999999999998},"max":{"1min":4.8440000000000003,"5min":4.8440000000000003,"15min":14.223000000000001},"last":1.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.1879999999999999,"15min":1.1339999999999999},"min":{"1min":0.84199999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":4.8789999999999996,"5min":4.8789999999999996,"15min":4.8789999999999996},"last":1.212},{"interface":"front","average":{"1min":1.391,"5min":1.236,"15min":1.2010000000000001},"min":{"1min":0.84599999999999997,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":1.1919999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2068,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11569,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:04:11.239 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:04:11.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:11 smithi153 bash[59565]: audit 2023-10-09T05:04:10.245997+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:11.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:11 smithi082 bash[77063]: audit 2023-10-09T05:04:10.245997+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:11.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:11 smithi082 bash[79733]: audit 2023-10-09T05:04:10.245997+0000 mgr.y (mgr.34107) 1509 : audit 0 from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:12.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:12 smithi153 bash[59565]: cluster 2023-10-09T05:04:11.082699+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:12.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:12 smithi082 bash[77063]: cluster 2023-10-09T05:04:11.082699+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:12.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:12 smithi082 bash[79733]: cluster 2023-10-09T05:04:11.082699+0000 mgr.y (mgr.34107) 1510 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:14.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:14 smithi153 bash[59565]: cluster 2023-10-09T05:04:13.083459+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:14.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:14 smithi082 bash[77063]: cluster 2023-10-09T05:04:13.083459+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:14.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:14 smithi082 bash[79733]: cluster 2023-10-09T05:04:13.083459+0000 mgr.y (mgr.34107) 1511 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:15.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:15 smithi153 bash[59565]: audit 2023-10-09T05:04:14.317034+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:15.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:15 smithi082 bash[77063]: audit 2023-10-09T05:04:14.317034+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:15.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:15 smithi082 bash[79733]: audit 2023-10-09T05:04:14.317034+0000 mon.a (mon.0) 2117 : audit 0 from='mgr.34107 172.21.15.82:0/290786291' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:16.021 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:16.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:16 smithi153 bash[59565]: cluster 2023-10-09T05:04:15.084244+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:16.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:16 smithi082 bash[77063]: cluster 2023-10-09T05:04:15.084244+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:16.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:16 smithi082 bash[79733]: cluster 2023-10-09T05:04:15.084244+0000 mgr.y (mgr.34107) 1512 : cluster 0 pgmap v1704: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:18.425 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:18 smithi153 bash[59565]: cluster 2023-10-09T05:04:17.085145+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:18.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:18 smithi082 bash[77063]: cluster 2023-10-09T05:04:17.085145+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:18.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:18 smithi082 bash[79733]: cluster 2023-10-09T05:04:17.085145+0000 mgr.y (mgr.34107) 1513 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:19.819 INFO:teuthology.orchestra.run.smithi082.stdout: 2023-10-09T05:04:19.821 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2023-10-09T05:04:19.981 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:19 smithi082 bash[73210]: ::ffff:172.21.15.153 - - [09/Oct/2023:05:04:19] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T05:04:20.424 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:20 smithi153 bash[59565]: cluster 2023-10-09T05:04:19.085911+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:20.481 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:20 smithi082 bash[77063]: cluster 2023-10-09T05:04:19.085911+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:20.481 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:20 smithi082 bash[79733]: cluster 2023-10-09T05:04:19.085911+0000 mgr.y (mgr.34107) 1514 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T05:04:20.657 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":1706,"stamp":"2023-10-09T05:04:19.085406+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12285,"num_read_kb":25974,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":12592450,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5971332,"kb_used_data":56196,"kb_used_omap":60,"kb_used_meta":5915011,"kb_avail":743989884,"statfs":{"total":767960285184,"available":761845641216,"internally_reserved":0,"allocated":57544704,"data_stored":49396043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62376,"internal_metadata":6056971352},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005040"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487363+0000","last_change":"2023-10-09T04:56:46.364962+0000","last_active":"2023-10-09T05:02:38.487363+0000","last_peered":"2023-10-09T05:02:38.487363+0000","last_clean":"2023-10-09T05:02:38.487363+0000","last_became_active":"2023-10-09T04:56:46.363050+0000","last_became_peered":"2023-10-09T04:56:46.363050+0000","last_unstale":"2023-10-09T05:02:38.487363+0000","last_undegraded":"2023-10-09T05:02:38.487363+0000","last_fullsized":"2023-10-09T05:02:38.487363+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":307,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111352+0000","last_change":"2023-10-09T05:01:06.296446+0000","last_active":"2023-10-09T05:02:48.111352+0000","last_peered":"2023-10-09T05:02:48.111352+0000","last_clean":"2023-10-09T05:02:48.111352+0000","last_became_active":"2023-10-09T05:01:06.295944+0000","last_became_peered":"2023-10-09T05:01:06.295944+0000","last_unstale":"2023-10-09T05:02:48.111352+0000","last_undegraded":"2023-10-09T05:02:48.111352+0000","last_fullsized":"2023-10-09T05:02:48.111352+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:14.139287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":306,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711753+0000","last_change":"2023-10-09T05:00:46.394380+0000","last_active":"2023-10-09T05:02:49.711753+0000","last_peered":"2023-10-09T05:02:49.711753+0000","last_clean":"2023-10-09T05:02:49.711753+0000","last_became_active":"2023-10-09T05:00:46.392900+0000","last_became_peered":"2023-10-09T05:00:46.392900+0000","last_unstale":"2023-10-09T05:02:49.711753+0000","last_undegraded":"2023-10-09T05:02:49.711753+0000","last_fullsized":"2023-10-09T05:02:49.711753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:00.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":381,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488416+0000","last_change":"2023-10-09T04:59:47.300587+0000","last_active":"2023-10-09T05:02:38.488416+0000","last_peered":"2023-10-09T05:02:38.488416+0000","last_clean":"2023-10-09T05:02:38.488416+0000","last_became_active":"2023-10-09T04:59:47.300212+0000","last_became_peered":"2023-10-09T04:59:47.300212+0000","last_unstale":"2023-10-09T05:02:38.488416+0000","last_undegraded":"2023-10-09T05:02:38.488416+0000","last_fullsized":"2023-10-09T05:02:38.488416+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:03.609661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109872+0000","last_change":"2023-10-09T05:01:06.808235+0000","last_active":"2023-10-09T05:02:48.109872+0000","last_peered":"2023-10-09T05:02:48.109872+0000","last_clean":"2023-10-09T05:02:48.109872+0000","last_became_active":"2023-10-09T05:01:06.807871+0000","last_became_peered":"2023-10-09T05:01:06.807871+0000","last_unstale":"2023-10-09T05:02:48.109872+0000","last_undegraded":"2023-10-09T05:02:48.109872+0000","last_fullsized":"2023-10-09T05:02:48.109872+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:00.789115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":337,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488514+0000","last_change":"2023-10-09T04:34:36.646246+0000","last_active":"2023-10-09T05:02:38.488514+0000","last_peered":"2023-10-09T05:02:38.488514+0000","last_clean":"2023-10-09T05:02:38.488514+0000","last_became_active":"2023-10-09T04:34:36.645911+0000","last_became_peered":"2023-10-09T04:34:36.645911+0000","last_unstale":"2023-10-09T05:02:38.488514+0000","last_undegraded":"2023-10-09T05:02:38.488514+0000","last_fullsized":"2023-10-09T05:02:38.488514+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":360,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196799+0000","last_change":"2023-10-09T04:34:36.646134+0000","last_active":"2023-10-09T05:02:48.196799+0000","last_peered":"2023-10-09T05:02:48.196799+0000","last_clean":"2023-10-09T05:02:48.196799+0000","last_became_active":"2023-10-09T04:34:36.645389+0000","last_became_peered":"2023-10-09T04:34:36.645389+0000","last_unstale":"2023-10-09T05:02:48.196799+0000","last_undegraded":"2023-10-09T05:02:48.196799+0000","last_fullsized":"2023-10-09T05:02:48.196799+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:26.740097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":306,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.294901+0000","last_change":"2023-10-09T04:41:21.461234+0000","last_active":"2023-10-09T05:02:38.294901+0000","last_peered":"2023-10-09T05:02:38.294901+0000","last_clean":"2023-10-09T05:02:38.294901+0000","last_became_active":"2023-10-09T04:41:21.460352+0000","last_became_peered":"2023-10-09T04:41:21.460352+0000","last_unstale":"2023-10-09T05:02:38.294901+0000","last_undegraded":"2023-10-09T05:02:38.294901+0000","last_fullsized":"2023-10-09T05:02:38.294901+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:42.340165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":338,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.309025+0000","last_change":"2023-10-09T04:59:47.299507+0000","last_active":"2023-10-09T05:02:38.309025+0000","last_peered":"2023-10-09T05:02:38.309025+0000","last_clean":"2023-10-09T05:02:38.309025+0000","last_became_active":"2023-10-09T04:59:47.299117+0000","last_became_peered":"2023-10-09T04:59:47.299117+0000","last_unstale":"2023-10-09T05:02:38.309025+0000","last_undegraded":"2023-10-09T05:02:38.309025+0000","last_fullsized":"2023-10-09T05:02:38.309025+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":162,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109893+0000","last_change":"2023-10-09T05:01:06.299381+0000","last_active":"2023-10-09T05:02:48.109893+0000","last_peered":"2023-10-09T05:02:48.109893+0000","last_clean":"2023-10-09T05:02:48.109893+0000","last_became_active":"2023-10-09T05:01:06.298866+0000","last_became_peered":"2023-10-09T05:01:06.298866+0000","last_unstale":"2023-10-09T05:02:48.109893+0000","last_undegraded":"2023-10-09T05:02:48.109893+0000","last_fullsized":"2023-10-09T05:02:48.109893+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:21.291886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":200,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489907+0000","last_change":"2023-10-09T04:59:47.304143+0000","last_active":"2023-10-09T05:02:38.489907+0000","last_peered":"2023-10-09T05:02:38.489907+0000","last_clean":"2023-10-09T05:02:38.489907+0000","last_became_active":"2023-10-09T04:59:47.303206+0000","last_became_peered":"2023-10-09T04:59:47.303206+0000","last_unstale":"2023-10-09T05:02:38.489907+0000","last_undegraded":"2023-10-09T05:02:38.489907+0000","last_fullsized":"2023-10-09T05:02:38.489907+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711753+0000","last_change":"2023-10-09T05:00:47.265174+0000","last_active":"2023-10-09T05:02:49.711753+0000","last_peered":"2023-10-09T05:02:49.711753+0000","last_clean":"2023-10-09T05:02:49.711753+0000","last_became_active":"2023-10-09T05:00:47.263977+0000","last_became_peered":"2023-10-09T05:00:47.263977+0000","last_unstale":"2023-10-09T05:02:49.711753+0000","last_undegraded":"2023-10-09T05:02:49.711753+0000","last_fullsized":"2023-10-09T05:02:49.711753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:30:48.497589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":103,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490277+0000","last_change":"2023-10-09T04:59:47.303414+0000","last_active":"2023-10-09T05:02:38.490277+0000","last_peered":"2023-10-09T05:02:38.490277+0000","last_clean":"2023-10-09T05:02:38.490277+0000","last_became_active":"2023-10-09T04:59:47.303003+0000","last_became_peered":"2023-10-09T04:59:47.303003+0000","last_unstale":"2023-10-09T05:02:38.490277+0000","last_undegraded":"2023-10-09T05:02:38.490277+0000","last_fullsized":"2023-10-09T05:02:38.490277+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490931+0000","last_change":"2023-10-09T04:56:46.360074+0000","last_active":"2023-10-09T05:02:38.490931+0000","last_peered":"2023-10-09T05:02:38.490931+0000","last_clean":"2023-10-09T05:02:38.490931+0000","last_became_active":"2023-10-09T04:56:46.359135+0000","last_became_peered":"2023-10-09T04:56:46.359135+0000","last_unstale":"2023-10-09T05:02:38.490931+0000","last_undegraded":"2023-10-09T05:02:38.490931+0000","last_fullsized":"2023-10-09T05:02:38.490931+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":215,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489302+0000","last_change":"2023-10-09T04:59:47.301222+0000","last_active":"2023-10-09T05:02:38.489302+0000","last_peered":"2023-10-09T05:02:38.489302+0000","last_clean":"2023-10-09T05:02:38.489302+0000","last_became_active":"2023-10-09T04:59:47.300692+0000","last_became_peered":"2023-10-09T04:59:47.300692+0000","last_unstale":"2023-10-09T05:02:38.489302+0000","last_undegraded":"2023-10-09T05:02:38.489302+0000","last_fullsized":"2023-10-09T05:02:38.489302+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110550+0000","last_change":"2023-10-09T05:01:06.808273+0000","last_active":"2023-10-09T05:02:48.110550+0000","last_peered":"2023-10-09T05:02:48.110550+0000","last_clean":"2023-10-09T05:02:48.110550+0000","last_became_active":"2023-10-09T05:01:06.807907+0000","last_became_peered":"2023-10-09T05:01:06.807907+0000","last_unstale":"2023-10-09T05:02:48.110550+0000","last_undegraded":"2023-10-09T05:02:48.110550+0000","last_fullsized":"2023-10-09T05:02:48.110550+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:47.665500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":378,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487207+0000","last_change":"2023-10-09T04:59:47.300887+0000","last_active":"2023-10-09T05:02:38.487207+0000","last_peered":"2023-10-09T05:02:38.487207+0000","last_clean":"2023-10-09T05:02:38.487207+0000","last_became_active":"2023-10-09T04:59:47.300489+0000","last_became_peered":"2023-10-09T04:59:47.300489+0000","last_unstale":"2023-10-09T05:02:38.487207+0000","last_undegraded":"2023-10-09T05:02:38.487207+0000","last_fullsized":"2023-10-09T05:02:38.487207+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:17.390470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":356,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.309003+0000","last_change":"2023-10-09T04:59:47.306310+0000","last_active":"2023-10-09T05:02:38.309003+0000","last_peered":"2023-10-09T05:02:38.309003+0000","last_clean":"2023-10-09T05:02:38.309003+0000","last_became_active":"2023-10-09T04:59:47.304895+0000","last_became_peered":"2023-10-09T04:59:47.304895+0000","last_unstale":"2023-10-09T05:02:38.309003+0000","last_undegraded":"2023-10-09T05:02:38.309003+0000","last_fullsized":"2023-10-09T05:02:38.309003+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488458+0000","last_change":"2023-10-09T04:59:47.305563+0000","last_active":"2023-10-09T05:02:38.488458+0000","last_peered":"2023-10-09T05:02:38.488458+0000","last_clean":"2023-10-09T05:02:38.488458+0000","last_became_active":"2023-10-09T04:59:47.304464+0000","last_became_peered":"2023-10-09T04:59:47.304464+0000","last_unstale":"2023-10-09T05:02:38.488458+0000","last_undegraded":"2023-10-09T05:02:38.488458+0000","last_fullsized":"2023-10-09T05:02:38.488458+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":378,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196732+0000","last_change":"2023-10-09T04:59:47.300755+0000","last_active":"2023-10-09T05:02:48.196732+0000","last_peered":"2023-10-09T05:02:48.196732+0000","last_clean":"2023-10-09T05:02:48.196732+0000","last_became_active":"2023-10-09T04:59:47.300264+0000","last_became_peered":"2023-10-09T04:59:47.300264+0000","last_unstale":"2023-10-09T05:02:48.196732+0000","last_undegraded":"2023-10-09T05:02:48.196732+0000","last_fullsized":"2023-10-09T05:02:48.196732+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:38.965025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":346,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488016+0000","last_change":"2023-10-09T04:34:36.638341+0000","last_active":"2023-10-09T05:02:38.488016+0000","last_peered":"2023-10-09T05:02:38.488016+0000","last_clean":"2023-10-09T05:02:38.488016+0000","last_became_active":"2023-10-09T04:34:36.637790+0000","last_became_peered":"2023-10-09T04:34:36.637790+0000","last_unstale":"2023-10-09T05:02:38.488016+0000","last_undegraded":"2023-10-09T05:02:38.488016+0000","last_fullsized":"2023-10-09T05:02:38.488016+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":294,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111432+0000","last_change":"2023-10-09T04:59:47.294816+0000","last_active":"2023-10-09T05:02:48.111432+0000","last_peered":"2023-10-09T05:02:48.111432+0000","last_clean":"2023-10-09T05:02:48.111432+0000","last_became_active":"2023-10-09T04:59:47.294405+0000","last_became_peered":"2023-10-09T04:59:47.294405+0000","last_unstale":"2023-10-09T05:02:48.111432+0000","last_undegraded":"2023-10-09T05:02:48.111432+0000","last_fullsized":"2023-10-09T05:02:48.111432+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:29.691664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":194,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489965+0000","last_change":"2023-10-09T04:59:47.307447+0000","last_active":"2023-10-09T05:02:38.489965+0000","last_peered":"2023-10-09T05:02:38.489965+0000","last_clean":"2023-10-09T05:02:38.489965+0000","last_became_active":"2023-10-09T04:59:47.305940+0000","last_became_peered":"2023-10-09T04:59:47.305940+0000","last_unstale":"2023-10-09T05:02:38.489965+0000","last_undegraded":"2023-10-09T05:02:38.489965+0000","last_fullsized":"2023-10-09T05:02:38.489965+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":329,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308459+0000","last_change":"2023-10-09T04:59:47.303400+0000","last_active":"2023-10-09T05:02:38.308459+0000","last_peered":"2023-10-09T05:02:38.308459+0000","last_clean":"2023-10-09T05:02:38.308459+0000","last_became_active":"2023-10-09T04:59:47.303065+0000","last_became_peered":"2023-10-09T04:59:47.303065+0000","last_unstale":"2023-10-09T05:02:38.308459+0000","last_undegraded":"2023-10-09T05:02:38.308459+0000","last_fullsized":"2023-10-09T05:02:38.308459+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":207,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489627+0000","last_change":"2023-10-09T04:59:47.301603+0000","last_active":"2023-10-09T05:02:38.489627+0000","last_peered":"2023-10-09T05:02:38.489627+0000","last_clean":"2023-10-09T05:02:38.489627+0000","last_became_active":"2023-10-09T04:59:47.300847+0000","last_became_peered":"2023-10-09T04:59:47.300847+0000","last_unstale":"2023-10-09T05:02:38.489627+0000","last_undegraded":"2023-10-09T05:02:38.489627+0000","last_fullsized":"2023-10-09T05:02:38.489627+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.486894+0000","last_change":"2023-10-09T04:56:46.351680+0000","last_active":"2023-10-09T05:02:38.486894+0000","last_peered":"2023-10-09T05:02:38.486894+0000","last_clean":"2023-10-09T05:02:38.486894+0000","last_became_active":"2023-10-09T04:56:46.351117+0000","last_became_peered":"2023-10-09T04:56:46.351117+0000","last_unstale":"2023-10-09T05:02:38.486894+0000","last_undegraded":"2023-10-09T05:02:38.486894+0000","last_fullsized":"2023-10-09T05:02:38.486894+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":174,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489711+0000","last_change":"2023-10-09T04:59:47.302428+0000","last_active":"2023-10-09T05:02:38.489711+0000","last_peered":"2023-10-09T05:02:38.489711+0000","last_clean":"2023-10-09T05:02:38.489711+0000","last_became_active":"2023-10-09T04:59:47.300886+0000","last_became_peered":"2023-10-09T04:59:47.300886+0000","last_unstale":"2023-10-09T05:02:38.489711+0000","last_undegraded":"2023-10-09T05:02:38.489711+0000","last_fullsized":"2023-10-09T05:02:38.489711+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":381,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488119+0000","last_change":"2023-10-09T04:59:47.299651+0000","last_active":"2023-10-09T05:02:38.488119+0000","last_peered":"2023-10-09T05:02:38.488119+0000","last_clean":"2023-10-09T05:02:38.488119+0000","last_became_active":"2023-10-09T04:59:47.299314+0000","last_became_peered":"2023-10-09T04:59:47.299314+0000","last_unstale":"2023-10-09T05:02:38.488119+0000","last_undegraded":"2023-10-09T05:02:38.488119+0000","last_fullsized":"2023-10-09T05:02:38.488119+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308743+0000","last_change":"2023-10-09T04:34:36.644438+0000","last_active":"2023-10-09T05:02:38.308743+0000","last_peered":"2023-10-09T05:02:38.308743+0000","last_clean":"2023-10-09T05:02:38.308743+0000","last_became_active":"2023-10-09T04:34:36.643945+0000","last_became_peered":"2023-10-09T04:34:36.643945+0000","last_unstale":"2023-10-09T05:02:38.308743+0000","last_undegraded":"2023-10-09T05:02:38.308743+0000","last_fullsized":"2023-10-09T05:02:38.308743+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":392,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:47.855465+0000","last_change":"2023-10-09T04:56:46.352313+0000","last_active":"2023-10-09T05:02:47.855465+0000","last_peered":"2023-10-09T05:02:47.855465+0000","last_clean":"2023-10-09T05:02:47.855465+0000","last_became_active":"2023-10-09T04:56:46.351505+0000","last_became_peered":"2023-10-09T04:56:46.351505+0000","last_unstale":"2023-10-09T05:02:47.855465+0000","last_undegraded":"2023-10-09T05:02:47.855465+0000","last_fullsized":"2023-10-09T05:02:47.855465+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:01.025392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":108,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110185+0000","last_change":"2023-10-09T05:01:06.296703+0000","last_active":"2023-10-09T05:02:48.110185+0000","last_peered":"2023-10-09T05:02:48.110185+0000","last_clean":"2023-10-09T05:02:48.110185+0000","last_became_active":"2023-10-09T05:01:06.296179+0000","last_became_peered":"2023-10-09T05:01:06.296179+0000","last_unstale":"2023-10-09T05:02:48.110185+0000","last_undegraded":"2023-10-09T05:02:48.110185+0000","last_fullsized":"2023-10-09T05:02:48.110185+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:40.351019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487007+0000","last_change":"2023-10-09T04:56:46.359130+0000","last_active":"2023-10-09T05:02:38.487007+0000","last_peered":"2023-10-09T05:02:38.487007+0000","last_clean":"2023-10-09T05:02:38.487007+0000","last_became_active":"2023-10-09T04:56:46.358781+0000","last_became_peered":"2023-10-09T04:56:46.358781+0000","last_unstale":"2023-10-09T05:02:38.487007+0000","last_undegraded":"2023-10-09T05:02:38.487007+0000","last_fullsized":"2023-10-09T05:02:38.487007+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":367,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487978+0000","last_change":"2023-10-09T04:56:46.360890+0000","last_active":"2023-10-09T05:02:38.487978+0000","last_peered":"2023-10-09T05:02:38.487978+0000","last_clean":"2023-10-09T05:02:38.487978+0000","last_became_active":"2023-10-09T04:56:46.360526+0000","last_became_peered":"2023-10-09T04:56:46.360526+0000","last_unstale":"2023-10-09T05:02:38.487978+0000","last_undegraded":"2023-10-09T05:02:38.487978+0000","last_fullsized":"2023-10-09T05:02:38.487978+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487319+0000","last_change":"2023-10-09T04:56:46.366317+0000","last_active":"2023-10-09T05:02:38.487319+0000","last_peered":"2023-10-09T05:02:38.487319+0000","last_clean":"2023-10-09T05:02:38.487319+0000","last_became_active":"2023-10-09T04:56:46.363159+0000","last_became_peered":"2023-10-09T04:56:46.363159+0000","last_unstale":"2023-10-09T05:02:38.487319+0000","last_undegraded":"2023-10-09T05:02:38.487319+0000","last_fullsized":"2023-10-09T05:02:38.487319+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489814+0000","last_change":"2023-10-09T04:59:47.306294+0000","last_active":"2023-10-09T05:02:38.489814+0000","last_peered":"2023-10-09T05:02:38.489814+0000","last_clean":"2023-10-09T05:02:38.489814+0000","last_became_active":"2023-10-09T04:59:47.305885+0000","last_became_peered":"2023-10-09T04:59:47.305885+0000","last_unstale":"2023-10-09T05:02:38.489814+0000","last_undegraded":"2023-10-09T05:02:38.489814+0000","last_fullsized":"2023-10-09T05:02:38.489814+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490743+0000","last_change":"2023-10-09T04:56:46.365379+0000","last_active":"2023-10-09T05:02:38.490743+0000","last_peered":"2023-10-09T05:02:38.490743+0000","last_clean":"2023-10-09T05:02:38.490743+0000","last_became_active":"2023-10-09T04:56:46.365000+0000","last_became_peered":"2023-10-09T04:56:46.365000+0000","last_unstale":"2023-10-09T05:02:38.490743+0000","last_undegraded":"2023-10-09T05:02:38.490743+0000","last_fullsized":"2023-10-09T05:02:38.490743+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109667+0000","last_change":"2023-10-09T04:59:47.303697+0000","last_active":"2023-10-09T05:02:48.109667+0000","last_peered":"2023-10-09T05:02:48.109667+0000","last_clean":"2023-10-09T05:02:48.109667+0000","last_became_active":"2023-10-09T04:59:47.303329+0000","last_became_peered":"2023-10-09T04:59:47.303329+0000","last_unstale":"2023-10-09T05:02:48.109667+0000","last_undegraded":"2023-10-09T05:02:48.109667+0000","last_fullsized":"2023-10-09T05:02:48.109667+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:38.849941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":256,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712766+0000","last_change":"2023-10-09T05:00:46.394373+0000","last_active":"2023-10-09T05:02:49.712766+0000","last_peered":"2023-10-09T05:02:49.712766+0000","last_clean":"2023-10-09T05:02:49.712766+0000","last_became_active":"2023-10-09T05:00:46.393892+0000","last_became_peered":"2023-10-09T05:00:46.393892+0000","last_unstale":"2023-10-09T05:02:49.712766+0000","last_undegraded":"2023-10-09T05:02:49.712766+0000","last_fullsized":"2023-10-09T05:02:49.712766+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:28.124258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"354'230","reported_seq":1012,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712354+0000","last_change":"2023-10-09T05:00:46.397813+0000","last_active":"2023-10-09T05:02:49.712354+0000","last_peered":"2023-10-09T05:02:49.712354+0000","last_clean":"2023-10-09T05:02:49.712354+0000","last_became_active":"2023-10-09T05:00:46.396936+0000","last_became_peered":"2023-10-09T05:00:46.396936+0000","last_unstale":"2023-10-09T05:02:49.712354+0000","last_undegraded":"2023-10-09T05:02:49.712354+0000","last_fullsized":"2023-10-09T05:02:49.712354+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":230,"log_dups_size":0,"ondisk_log_size":230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:00.513299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1269,"num_read_kb":2981,"num_write":220,"num_write_kb":1540,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308930+0000","last_change":"2023-10-09T04:44:51.887494+0000","last_active":"2023-10-09T05:02:38.308930+0000","last_peered":"2023-10-09T05:02:38.308930+0000","last_clean":"2023-10-09T05:02:38.308930+0000","last_became_active":"2023-10-09T04:44:51.886703+0000","last_became_peered":"2023-10-09T04:44:51.886703+0000","last_unstale":"2023-10-09T05:02:38.308930+0000","last_undegraded":"2023-10-09T05:02:38.308930+0000","last_fullsized":"2023-10-09T05:02:38.308930+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":275,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711800+0000","last_change":"2023-10-09T05:00:46.395242+0000","last_active":"2023-10-09T05:02:49.711800+0000","last_peered":"2023-10-09T05:02:49.711800+0000","last_clean":"2023-10-09T05:02:49.711800+0000","last_became_active":"2023-10-09T05:00:46.394902+0000","last_became_peered":"2023-10-09T05:00:46.394902+0000","last_unstale":"2023-10-09T05:02:49.711800+0000","last_undegraded":"2023-10-09T05:02:49.711800+0000","last_fullsized":"2023-10-09T05:02:49.711800+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:24.054556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"354'114","reported_seq":1144,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308618+0000","last_change":"2023-10-09T04:59:14.322562+0000","last_active":"2023-10-09T05:02:38.308618+0000","last_peered":"2023-10-09T05:02:38.308618+0000","last_clean":"2023-10-09T05:02:38.308618+0000","last_became_active":"2023-10-09T04:34:36.643706+0000","last_became_peered":"2023-10-09T04:34:36.643706+0000","last_unstale":"2023-10-09T05:02:38.308618+0000","last_undegraded":"2023-10-09T05:02:38.308618+0000","last_fullsized":"2023-10-09T05:02:38.308618+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1066,"num_read_kb":3635,"num_write":105,"num_write_kb":2407,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490767+0000","last_change":"2023-10-09T04:56:46.360770+0000","last_active":"2023-10-09T05:02:38.490767+0000","last_peered":"2023-10-09T05:02:38.490767+0000","last_clean":"2023-10-09T05:02:38.490767+0000","last_became_active":"2023-10-09T04:56:46.360413+0000","last_became_peered":"2023-10-09T04:56:46.360413+0000","last_unstale":"2023-10-09T05:02:38.490767+0000","last_undegraded":"2023-10-09T05:02:38.490767+0000","last_fullsized":"2023-10-09T05:02:38.490767+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.711881+0000","last_change":"2023-10-09T04:57:17.362172+0000","last_active":"2023-10-09T05:02:49.711881+0000","last_peered":"2023-10-09T05:02:49.711881+0000","last_clean":"2023-10-09T05:02:49.711881+0000","last_became_active":"2023-10-09T04:57:17.361793+0000","last_became_peered":"2023-10-09T04:57:17.361793+0000","last_unstale":"2023-10-09T05:02:49.711881+0000","last_undegraded":"2023-10-09T05:02:49.711881+0000","last_fullsized":"2023-10-09T05:02:49.711881+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:18.548269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"354'346","reported_seq":1501,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.297086+0000","last_change":"2023-10-09T05:02:29.587647+0000","last_active":"2023-10-09T05:02:38.297086+0000","last_peered":"2023-10-09T05:02:38.297086+0000","last_clean":"2023-10-09T05:02:38.297086+0000","last_became_active":"2023-10-09T05:02:29.586702+0000","last_became_peered":"2023-10-09T05:02:29.586702+0000","last_unstale":"2023-10-09T05:02:38.297086+0000","last_undegraded":"2023-10-09T05:02:38.297086+0000","last_fullsized":"2023-10-09T05:02:38.297086+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":346,"log_dups_size":0,"ondisk_log_size":346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:03:34.935284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071613799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2556,"num_read_kb":3486,"num_write":298,"num_write_kb":1370,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":287,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.040666+0000","last_change":"2023-10-09T05:00:47.265150+0000","last_active":"2023-10-09T05:02:49.040666+0000","last_peered":"2023-10-09T05:02:49.040666+0000","last_clean":"2023-10-09T05:02:49.040666+0000","last_became_active":"2023-10-09T05:00:47.264137+0000","last_became_peered":"2023-10-09T05:00:47.264137+0000","last_unstale":"2023-10-09T05:02:49.040666+0000","last_undegraded":"2023-10-09T05:02:49.040666+0000","last_fullsized":"2023-10-09T05:02:49.040666+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:34.580840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":301,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712730+0000","last_change":"2023-10-09T05:00:46.395582+0000","last_active":"2023-10-09T05:02:49.712730+0000","last_peered":"2023-10-09T05:02:49.712730+0000","last_clean":"2023-10-09T05:02:49.712730+0000","last_became_active":"2023-10-09T05:00:46.395343+0000","last_became_peered":"2023-10-09T05:00:46.395343+0000","last_unstale":"2023-10-09T05:02:49.712730+0000","last_undegraded":"2023-10-09T05:02:49.712730+0000","last_fullsized":"2023-10-09T05:02:49.712730+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:50.360234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"354'533","reported_seq":2206,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308614+0000","last_change":"2023-10-09T04:59:50.323684+0000","last_active":"2023-10-09T05:02:38.308614+0000","last_peered":"2023-10-09T05:02:38.308614+0000","last_clean":"2023-10-09T05:02:38.308614+0000","last_became_active":"2023-10-09T04:59:50.320710+0000","last_became_peered":"2023-10-09T04:59:50.320710+0000","last_unstale":"2023-10-09T05:02:38.308614+0000","last_undegraded":"2023-10-09T05:02:38.308614+0000","last_fullsized":"2023-10-09T05:02:38.308614+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":533,"log_dups_size":0,"ondisk_log_size":533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2972,"num_read_kb":4653,"num_write":476,"num_write_kb":6413,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":577,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488399+0000","last_change":"2023-10-09T04:59:47.308170+0000","last_active":"2023-10-09T05:02:38.488399+0000","last_peered":"2023-10-09T05:02:38.488399+0000","last_clean":"2023-10-09T05:02:38.488399+0000","last_became_active":"2023-10-09T04:59:47.307161+0000","last_became_peered":"2023-10-09T04:59:47.307161+0000","last_unstale":"2023-10-09T05:02:38.488399+0000","last_undegraded":"2023-10-09T05:02:38.488399+0000","last_fullsized":"2023-10-09T05:02:38.488399+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:05:09.485147+0000","last_clean_scrub_stamp":"2023-10-09T04:05:09.485147+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.110099+0000","last_change":"2023-10-09T05:01:06.298992+0000","last_active":"2023-10-09T05:02:48.110099+0000","last_peered":"2023-10-09T05:02:48.110099+0000","last_clean":"2023-10-09T05:02:48.110099+0000","last_became_active":"2023-10-09T05:01:06.298680+0000","last_became_peered":"2023-10-09T05:01:06.298680+0000","last_unstale":"2023-10-09T05:02:48.110099+0000","last_undegraded":"2023-10-09T05:02:48.110099+0000","last_fullsized":"2023-10-09T05:02:48.110099+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:57.686146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488802+0000","last_change":"2023-10-09T04:59:47.305097+0000","last_active":"2023-10-09T05:02:38.488802+0000","last_peered":"2023-10-09T05:02:38.488802+0000","last_clean":"2023-10-09T05:02:38.488802+0000","last_became_active":"2023-10-09T04:59:47.304738+0000","last_became_peered":"2023-10-09T04:59:47.304738+0000","last_unstale":"2023-10-09T05:02:38.488802+0000","last_undegraded":"2023-10-09T05:02:38.488802+0000","last_fullsized":"2023-10-09T05:02:38.488802+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"354'224","reported_seq":1176,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488951+0000","last_change":"2023-10-09T04:59:50.317479+0000","last_active":"2023-10-09T05:02:38.488951+0000","last_peered":"2023-10-09T05:02:38.488951+0000","last_clean":"2023-10-09T05:02:38.488951+0000","last_became_active":"2023-10-09T04:59:50.313715+0000","last_became_peered":"2023-10-09T04:59:50.313715+0000","last_unstale":"2023-10-09T05:02:38.488951+0000","last_undegraded":"2023-10-09T05:02:38.488951+0000","last_fullsized":"2023-10-09T05:02:38.488951+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":224,"log_dups_size":0,"ondisk_log_size":224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1839,"num_read_kb":1902,"num_write":163,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":413,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196042+0000","last_change":"2023-10-09T04:56:46.362929+0000","last_active":"2023-10-09T05:02:48.196042+0000","last_peered":"2023-10-09T05:02:48.196042+0000","last_clean":"2023-10-09T05:02:48.196042+0000","last_became_active":"2023-10-09T04:56:46.362453+0000","last_became_peered":"2023-10-09T04:56:46.362453+0000","last_unstale":"2023-10-09T05:02:48.196042+0000","last_undegraded":"2023-10-09T05:02:48.196042+0000","last_fullsized":"2023-10-09T05:02:48.196042+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:34.707239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196534+0000","last_change":"2023-10-09T04:44:51.895754+0000","last_active":"2023-10-09T05:02:48.196534+0000","last_peered":"2023-10-09T05:02:48.196534+0000","last_clean":"2023-10-09T05:02:48.196534+0000","last_became_active":"2023-10-09T04:44:51.895253+0000","last_became_peered":"2023-10-09T04:44:51.895253+0000","last_unstale":"2023-10-09T05:02:48.196534+0000","last_undegraded":"2023-10-09T05:02:48.196534+0000","last_fullsized":"2023-10-09T05:02:48.196534+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:03.927492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"354'240","reported_seq":841,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489525+0000","last_change":"2023-10-09T04:59:50.319553+0000","last_active":"2023-10-09T05:02:38.489525+0000","last_peered":"2023-10-09T05:02:38.489525+0000","last_clean":"2023-10-09T05:02:38.489525+0000","last_became_active":"2023-10-09T04:59:50.314137+0000","last_became_peered":"2023-10-09T04:59:50.314137+0000","last_unstale":"2023-10-09T05:02:38.489525+0000","last_undegraded":"2023-10-09T05:02:38.489525+0000","last_fullsized":"2023-10-09T05:02:38.489525+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1239,"num_read_kb":4520,"num_write":217,"num_write_kb":8525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":287,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712244+0000","last_change":"2023-10-09T05:00:47.264985+0000","last_active":"2023-10-09T05:02:49.712244+0000","last_peered":"2023-10-09T05:02:49.712244+0000","last_clean":"2023-10-09T05:02:49.712244+0000","last_became_active":"2023-10-09T05:00:47.264057+0000","last_became_peered":"2023-10-09T05:00:47.264057+0000","last_unstale":"2023-10-09T05:02:49.712244+0000","last_undegraded":"2023-10-09T05:02:49.712244+0000","last_fullsized":"2023-10-09T05:02:49.712244+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:34.503675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":381,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196459+0000","last_change":"2023-10-09T04:59:47.302043+0000","last_active":"2023-10-09T05:02:48.196459+0000","last_peered":"2023-10-09T05:02:48.196459+0000","last_clean":"2023-10-09T05:02:48.196459+0000","last_became_active":"2023-10-09T04:59:47.301652+0000","last_became_peered":"2023-10-09T04:59:47.301652+0000","last_unstale":"2023-10-09T05:02:48.196459+0000","last_undegraded":"2023-10-09T05:02:48.196459+0000","last_fullsized":"2023-10-09T05:02:48.196459+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:01.559860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":108,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111491+0000","last_change":"2023-10-09T05:01:06.299766+0000","last_active":"2023-10-09T05:02:48.111491+0000","last_peered":"2023-10-09T05:02:48.111491+0000","last_clean":"2023-10-09T05:02:48.111491+0000","last_became_active":"2023-10-09T05:01:06.299339+0000","last_became_peered":"2023-10-09T05:01:06.299339+0000","last_unstale":"2023-10-09T05:02:48.111491+0000","last_undegraded":"2023-10-09T05:02:48.111491+0000","last_fullsized":"2023-10-09T05:02:48.111491+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:09.717433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"354'151","reported_seq":696,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:49.712733+0000","last_change":"2023-10-09T05:00:46.397918+0000","last_active":"2023-10-09T05:02:49.712733+0000","last_peered":"2023-10-09T05:02:49.712733+0000","last_clean":"2023-10-09T05:02:49.712733+0000","last_became_active":"2023-10-09T05:00:46.397421+0000","last_became_peered":"2023-10-09T05:00:46.397421+0000","last_unstale":"2023-10-09T05:02:49.712733+0000","last_undegraded":"2023-10-09T05:02:49.712733+0000","last_fullsized":"2023-10-09T05:02:49.712733+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":151,"log_dups_size":0,"ondisk_log_size":151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:14.581626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":691,"num_read_kb":2271,"num_write":97,"num_write_kb":1441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":162,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.109674+0000","last_change":"2023-10-09T05:01:06.299585+0000","last_active":"2023-10-09T05:02:48.109674+0000","last_peered":"2023-10-09T05:02:48.109674+0000","last_clean":"2023-10-09T05:02:48.109674+0000","last_became_active":"2023-10-09T05:01:06.299190+0000","last_became_peered":"2023-10-09T05:01:06.299190+0000","last_unstale":"2023-10-09T05:02:48.109674+0000","last_undegraded":"2023-10-09T05:02:48.109674+0000","last_fullsized":"2023-10-09T05:02:48.109674+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:06:53.232841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"354'112","reported_seq":693,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.488663+0000","last_change":"2023-10-09T04:59:47.321093+0000","last_active":"2023-10-09T05:02:38.488663+0000","last_peered":"2023-10-09T05:02:38.488663+0000","last_clean":"2023-10-09T05:02:38.488663+0000","last_became_active":"2023-10-09T04:59:47.300409+0000","last_became_peered":"2023-10-09T04:59:47.300409+0000","last_unstale":"2023-10-09T05:02:38.488663+0000","last_undegraded":"2023-10-09T05:02:38.488663+0000","last_fullsized":"2023-10-09T05:02:38.488663+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:35.285998+0000","last_clean_scrub_stamp":"2023-10-09T04:09:35.285998+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":359,"num_read_kb":2026,"num_write":114,"num_write_kb":2481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.196605+0000","last_change":"2023-10-09T04:56:46.422892+0000","last_active":"2023-10-09T05:02:48.196605+0000","last_peered":"2023-10-09T05:02:48.196605+0000","last_clean":"2023-10-09T05:02:48.196605+0000","last_became_active":"2023-10-09T04:56:46.363249+0000","last_became_peered":"2023-10-09T04:56:46.363249+0000","last_unstale":"2023-10-09T05:02:48.196605+0000","last_undegraded":"2023-10-09T05:02:48.196605+0000","last_fullsized":"2023-10-09T05:02:48.196605+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:48.389583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487214+0000","last_change":"2023-10-09T04:56:46.358968+0000","last_active":"2023-10-09T05:02:38.487214+0000","last_peered":"2023-10-09T05:02:38.487214+0000","last_clean":"2023-10-09T05:02:38.487214+0000","last_became_active":"2023-10-09T04:56:46.358613+0000","last_became_peered":"2023-10-09T04:56:46.358613+0000","last_unstale":"2023-10-09T05:02:38.487214+0000","last_undegraded":"2023-10-09T05:02:38.487214+0000","last_fullsized":"2023-10-09T05:02:38.487214+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":384,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.487428+0000","last_change":"2023-10-09T04:59:47.301699+0000","last_active":"2023-10-09T05:02:38.487428+0000","last_peered":"2023-10-09T05:02:38.487428+0000","last_clean":"2023-10-09T05:02:38.487428+0000","last_became_active":"2023-10-09T04:59:47.301300+0000","last_became_peered":"2023-10-09T04:59:47.301300+0000","last_unstale":"2023-10-09T05:02:38.487428+0000","last_undegraded":"2023-10-09T05:02:38.487428+0000","last_fullsized":"2023-10-09T05:02:38.487428+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:57.351668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.297358+0000","last_change":"2023-10-09T05:02:29.582970+0000","last_active":"2023-10-09T05:02:38.297358+0000","last_peered":"2023-10-09T05:02:38.297358+0000","last_clean":"2023-10-09T05:02:38.297358+0000","last_became_active":"2023-10-09T05:02:29.582421+0000","last_became_peered":"2023-10-09T05:02:29.582421+0000","last_unstale":"2023-10-09T05:02:38.297358+0000","last_undegraded":"2023-10-09T05:02:38.297358+0000","last_fullsized":"2023-10-09T05:02:38.297358+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:24.256804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":333,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308935+0000","last_change":"2023-10-09T04:56:46.363704+0000","last_active":"2023-10-09T05:02:38.308935+0000","last_peered":"2023-10-09T05:02:38.308935+0000","last_clean":"2023-10-09T05:02:38.308935+0000","last_became_active":"2023-10-09T04:56:46.343047+0000","last_became_peered":"2023-10-09T04:56:46.343047+0000","last_unstale":"2023-10-09T05:02:38.308935+0000","last_undegraded":"2023-10-09T05:02:38.308935+0000","last_fullsized":"2023-10-09T05:02:38.308935+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.490090+0000","last_change":"2023-10-09T04:56:46.350221+0000","last_active":"2023-10-09T05:02:38.490090+0000","last_peered":"2023-10-09T05:02:38.490090+0000","last_clean":"2023-10-09T05:02:38.490090+0000","last_became_active":"2023-10-09T04:56:46.349461+0000","last_became_peered":"2023-10-09T04:56:46.349461+0000","last_unstale":"2023-10-09T05:02:38.490090+0000","last_undegraded":"2023-10-09T05:02:38.490090+0000","last_fullsized":"2023-10-09T05:02:38.490090+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":326,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308887+0000","last_change":"2023-10-09T04:56:46.375959+0000","last_active":"2023-10-09T05:02:38.308887+0000","last_peered":"2023-10-09T05:02:38.308887+0000","last_clean":"2023-10-09T05:02:38.308887+0000","last_became_active":"2023-10-09T04:56:46.354426+0000","last_became_peered":"2023-10-09T04:56:46.354426+0000","last_unstale":"2023-10-09T05:02:38.308887+0000","last_undegraded":"2023-10-09T05:02:38.308887+0000","last_fullsized":"2023-10-09T05:02:38.308887+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.486954+0000","last_change":"2023-10-09T04:56:46.362257+0000","last_active":"2023-10-09T05:02:38.486954+0000","last_peered":"2023-10-09T05:02:38.486954+0000","last_clean":"2023-10-09T05:02:38.486954+0000","last_became_active":"2023-10-09T04:56:46.361599+0000","last_became_peered":"2023-10-09T04:56:46.361599+0000","last_unstale":"2023-10-09T05:02:38.486954+0000","last_undegraded":"2023-10-09T05:02:38.486954+0000","last_fullsized":"2023-10-09T05:02:38.486954+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":116,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.111265+0000","last_change":"2023-10-09T05:01:06.298300+0000","last_active":"2023-10-09T05:02:48.111265+0000","last_peered":"2023-10-09T05:02:48.111265+0000","last_clean":"2023-10-09T05:02:48.111265+0000","last_became_active":"2023-10-09T05:01:06.297968+0000","last_became_peered":"2023-10-09T05:01:06.297968+0000","last_unstale":"2023-10-09T05:02:48.111265+0000","last_undegraded":"2023-10-09T05:02:48.111265+0000","last_fullsized":"2023-10-09T05:02:48.111265+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:19.388221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.308455+0000","last_change":"2023-10-09T04:53:31.814454+0000","last_active":"2023-10-09T05:02:38.308455+0000","last_peered":"2023-10-09T05:02:38.308455+0000","last_clean":"2023-10-09T05:02:38.308455+0000","last_became_active":"2023-10-09T04:53:31.814010+0000","last_became_peered":"2023-10-09T04:53:31.814010+0000","last_unstale":"2023-10-09T05:02:38.308455+0000","last_undegraded":"2023-10-09T05:02:38.308455+0000","last_fullsized":"2023-10-09T05:02:38.308455+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":382,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T05:02:48.197046+0000","last_change":"2023-10-09T04:44:51.888210+0000","last_active":"2023-10-09T05:02:48.197046+0000","last_peered":"2023-10-09T05:02:48.197046+0000","last_clean":"2023-10-09T05:02:48.197046+0000","last_became_active":"2023-10-09T04:44:51.887712+0000","last_became_peered":"2023-10-09T04:44:51.887712+0000","last_unstale":"2023-10-09T05:02:48.197046+0000","last_undegraded":"2023-10-09T05:02:48.197046+0000","last_fullsized":"2023-10-09T05:02:48.197046+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:25.176558+0000","last_clean_scrub_stamp":"2023-10-09T04:11:25.176558+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:54.646897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-10-09T05:02:38.489312+0000","last_change":"2023-10-09T04:56:46.359415+0000","last_active":"2023-10-09T05:02:38.489312+0000","last_peered":"2023-10-09T05:02:38.489312+0000","last_clean":"2023-10-09T05:02:38.489312+0000","last_became_active":"2023-10-09T04:56:46.358569+0000","last_became_peered":"2023-10-09T04:56:46.358569+0000","last_unstale":"2023-10-09T05:02:38.489312+0000","last_undegraded":"2023-10-09T05:02:38.489312+0000","last_fullsized":"2023-10-09T05:02:38.489312+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:11:26.190073+0000","last_clean_scrub_stamp":"2023-10-09T04:11:26.190073+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":9538,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53892,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11991,"num_read_kb":25474,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8484,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":358,"seq":1537598292024,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377924,"kb_used_data":12092,"kb_used_omap":13,"kb_used_meta":365810,"kb_avail":93367228,"statfs":{"total":95995035648,"available":95608041472,"internally_reserved":0,"allocated":12382208,"data_stored":11735160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13637,"internal_metadata":374590139},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.96899999999999997,"15min":1.0309999999999999},"min":{"1min":0.63,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.137,"5min":1.486,"15min":1.486},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97399999999999998,"15min":0.95999999999999996},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.212,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.1499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.95799999999999996,"15min":0.98199999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.1519999999999999,"5min":1.444,"15min":1.444},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.95399999999999996,"15min":0.95799999999999996},"min":{"1min":0.65000000000000002,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.272,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.91200000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.98799999999999999,"15min":0.98699999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.242,"5min":1.482,"15min":1.482},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.002,"15min":0.99299999999999999},"min":{"1min":0.66300000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2350000000000001,"5min":1.599,"15min":1.599},"last":0.86399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0169999999999999,"15min":1.0249999999999999},"min":{"1min":0.72599999999999998,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.3220000000000001,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.048,"15min":1.0780000000000001},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.3340000000000001,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.202}]},{"osd":4,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.752,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.3169999999999999,"5min":1.734,"15min":1.734},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.98199999999999998,"15min":0.96099999999999997},"min":{"1min":0.77500000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2330000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.0840000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.028,"15min":1.0069999999999999},"min":{"1min":0.80100000000000005,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.3540000000000001,"5min":1.365,"15min":1.365},"last":1.038},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.073,"15min":1.0569999999999999},"min":{"1min":0.70599999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2949999999999999,"5min":1.667,"15min":1.667},"last":1.069}]},{"osd":6,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.0740000000000001,"15min":1.044},"min":{"1min":0.76000000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5049999999999999,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.145,"15min":1.127},"min":{"1min":0.879,"5min":0.372,"15min":0.372},"max":{"1min":1.448,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.173}]}]},{"osd":6,"up_from":146,"seq":627065225580,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618236,"kb_used_data":3892,"kb_used_omap":5,"kb_used_meta":614330,"kb_avail":93126916,"statfs":{"total":95995035648,"available":95361961984,"internally_reserved":0,"allocated":3985408,"data_stored":3346425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":629074528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.877,"15min":2.1360000000000001},"min":{"1min":0.54700000000000004,"5min":0.51800000000000002,"15min":0.246},"max":{"1min":1.29,"5min":1.5509999999999999,"15min":81.287999999999997},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.876,"5min":0.91900000000000004,"15min":1.8979999999999999},"min":{"1min":0.621,"5min":0.40600000000000003,"15min":0.254},"max":{"1min":1.151,"5min":1.5369999999999999,"15min":81.168999999999997},"last":0.78300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.91100000000000003,"15min":1.6399999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.28799999999999998},"max":{"1min":1.2270000000000001,"5min":1.2929999999999999,"15min":40.709000000000003},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.91100000000000003,"15min":1.7230000000000001},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.28299999999999997},"max":{"1min":1.194,"5min":1.5269999999999999,"15min":40.414000000000001},"last":1.0600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.92900000000000005,"15min":1.919},"min":{"1min":0.72499999999999998,"5min":0.59899999999999998,"15min":0.31900000000000001},"max":{"1min":1.258,"5min":1.464,"15min":97.284999999999997},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.94699999999999995,"15min":1.732},"min":{"1min":0.57599999999999996,"5min":0.55600000000000005,"15min":0.30599999999999999},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":97.069000000000003},"last":0.88600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97799999999999998,"15min":1.423},"min":{"1min":0.65600000000000003,"5min":0.54400000000000004,"15min":0.318},"max":{"1min":1.357,"5min":1.357,"15min":12.050000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0089999999999999,"15min":1.52},"min":{"1min":0.67400000000000004,"5min":0.496,"15min":0.34499999999999997},"max":{"1min":1.3440000000000001,"5min":1.54,"15min":18.387},"last":0.90900000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.97199999999999998,"15min":0.88900000000000001},"min":{"1min":0.64400000000000002,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":1.3640000000000001,"5min":1.5960000000000001,"15min":4.9139999999999997},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0429999999999999,"15min":1.216},"min":{"1min":0.59899999999999998,"5min":0.44900000000000001,"15min":0.34499999999999997},"max":{"1min":1.359,"5min":1.6459999999999999,"15min":10.428000000000001},"last":1.0189999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:03:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.046,"15min":1.2030000000000001},"min":{"1min":0.66600000000000004,"5min":0.38200000000000001,"15min":0.28000000000000003},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":29.077999999999999},"last":1.044},{"interface":"front","average":{"1min":1.181,"5min":1.0740000000000001,"15min":1.645},"min":{"1min":0.77900000000000003,"5min":0.41499999999999998,"15min":0.34999999999999998},"max":{"1min":1.4379999999999999,"5min":1.738,"15min":28.742000000000001},"last":1.093}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.083,"15min":1.0349999999999999},"min":{"1min":0.75600000000000001,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.4039999999999999,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.1479999999999999},{"interface":"front","average":{"1min":1.167,"5min":1.1659999999999999,"15min":1.1279999999999999},"min":{"1min":0.89300000000000002,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.524,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.232}]}]},{"osd":1,"up_from":93,"seq":399431959029,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917464,"kb_used_data":4112,"kb_used_omap":6,"kb_used_meta":913337,"kb_avail":92827688,"statfs":{"total":95995035648,"available":95055552512,"internally_reserved":0,"allocated":4210688,"data_stored":2852572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":935257100},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.68300000000000005,"15min":1.538},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.183},"max":{"1min":0.94999999999999996,"5min":1.1379999999999999,"15min":47.274000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.78900000000000003,"15min":1.3660000000000001},"min":{"1min":0.41099999999999998,"5min":0.314,"15min":0.23400000000000001},"max":{"1min":1.143,"5min":1.363,"15min":47.372999999999998},"last":1.0209999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:03:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.74399999999999999,"15min":1.1879999999999999},"min":{"1min":0.42599999999999999,"5min":0.34300000000000003,"15min":0.21299999999999999},"max":{"1min":1.784,"5min":1.925,"15min":10.004},"last":0.75},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.70499999999999996,"15min":1.3169999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.20999999999999999},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":15.048},"last":0.97999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:04:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.72799999999999998,"15min":1.7310000000000001},"min":{"1min":0.376,"5min":0.27600000000000002,"15min":0.23300000000000001},"max":{"1min":1.03,"5min":1.177,"15min":68.974000000000004},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.876,"15min":1.798},"min":{"1min":0.55800000000000005,"5min":0.38400000000000001,"15min":0.26000000000000001},"max":{"1min":1.3,"5min":1.3049999999999999,"15min":69.183000000000007},"last":1.2170000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:04:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.872,"15min":1.4079999999999999},"min":{"1min":0.52000000000000002,"5min":0.46000000000000002,"15min":0.308},"max":{"1min":1.1120000000000001,"5min":1.1639999999999999,"15min":10.398},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.95299999999999996,"15min":1.361},"min":{"1min":0.628,"5min":0.505,"15min":0.34499999999999997},"max":{"1min":1.3300000000000001,"5min":1.78,"15min":9.827},"last":1.0820000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.012,"15min":1.4199999999999999},"min":{"1min":0.70599999999999996,"5min":0.54300000000000004,"15min":0.32500000000000001},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":10.438000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.96199999999999997,"15min":1.5660000000000001},"min":{"1min":0.66900000000000004,"5min":0.55100000000000005,"15min":0.33100000000000002},"max":{"1min":1.3009999999999999,"5min":1.448,"15min":10.435},"last":1.173}]},{"osd":6,"last update":"Mon Oct 9 05:04:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.115,"15min":1.5009999999999999},"min":{"1min":0.64800000000000002,"5min":0.64300000000000002,"15min":0.35099999999999998},"max":{"1min":1.716,"5min":1.716,"15min":18.295000000000002},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.194,"15min":1.7629999999999999},"min":{"1min":0.872,"5min":0.69199999999999995,"15min":0.41999999999999998},"max":{"1min":1.647,"5min":1.647,"15min":68.581000000000003},"last":1.377}]},{"osd":7,"last update":"Mon Oct 9 05:03:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.1100000000000001,"15min":1.119},"min":{"1min":0.63800000000000001,"5min":0.624,"15min":0.624},"max":{"1min":1.407,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.415},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.1779999999999999,"15min":1.1759999999999999},"min":{"1min":0.81399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.5589999999999999,"5min":1.71,"15min":1.71},"last":1.462}]}]},{"osd":0,"up_from":88,"seq":377957122554,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946460,"kb_used_data":8852,"kb_used_omap":9,"kb_used_meta":937590,"kb_avail":92798692,"statfs":{"total":95995035648,"available":95025860608,"internally_reserved":0,"allocated":9064448,"data_stored":7650535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":960092657},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67300000000000004,"15min":1.4690000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.18099999999999999},"max":{"1min":1.054,"5min":1.2150000000000001,"15min":62.396000000000001},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.70399999999999996,"15min":1.571},"min":{"1min":0.39900000000000002,"5min":0.376,"15min":0.19400000000000001},"max":{"1min":1.0649999999999999,"5min":1.22,"15min":62.496000000000002},"last":0.92400000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.72699999999999998,"15min":1.113},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.23699999999999999},"max":{"1min":0.94499999999999995,"5min":2.8610000000000002,"15min":13.493},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.76200000000000001,"15min":1.3240000000000001},"min":{"1min":0.40400000000000003,"5min":0.38600000000000001,"15min":0.246},"max":{"1min":1.2270000000000001,"5min":1.2929999999999999,"15min":10.500999999999999},"last":0.83999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80500000000000005,"15min":1.349},"min":{"1min":0.377,"5min":0.377,"15min":0.192},"max":{"1min":1.333,"5min":1.4370000000000001,"15min":13.545999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.871,"5min":0.876,"15min":1.5760000000000001},"min":{"1min":0.54800000000000004,"5min":0.45200000000000001,"15min":0.21199999999999999},"max":{"1min":1.294,"5min":1.635,"15min":13.544},"last":0.98099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.92400000000000004,"15min":1.5389999999999999},"min":{"1min":0.66500000000000004,"5min":0.56499999999999995,"15min":0.33300000000000002},"max":{"1min":1.157,"5min":1.371,"15min":14.996},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0609999999999999,"15min":1.3180000000000001},"min":{"1min":0.79600000000000004,"5min":0.66300000000000003,"15min":0.32000000000000001},"max":{"1min":1.385,"5min":1.468,"15min":13.417},"last":1.143}]},{"osd":5,"last update":"Mon Oct 9 05:03:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.05,"15min":1.5089999999999999},"min":{"1min":0.77500000000000002,"5min":0.61299999999999999,"15min":0.33800000000000002},"max":{"1min":1.464,"5min":1.5780000000000001,"15min":10.621},"last":1.071},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.046,"15min":1.4870000000000001},"min":{"1min":0.68000000000000005,"5min":0.61799999999999999,"15min":0.32600000000000001},"max":{"1min":1.3899999999999999,"5min":1.5409999999999999,"15min":13.374000000000001},"last":1.3049999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.0980000000000001,"15min":1.448},"min":{"1min":0.74299999999999999,"5min":0.58599999999999997,"15min":0.33500000000000002},"max":{"1min":1.4099999999999999,"5min":1.581,"15min":13.445},"last":1.228},{"interface":"front","average":{"1min":1.046,"5min":1.1180000000000001,"15min":1.6439999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.38700000000000001},"max":{"1min":1.595,"5min":1.6970000000000001,"15min":13.477},"last":1.0289999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.1399999999999999,"15min":1.1859999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.4219999999999999,"5min":1.647,"15min":1.647},"last":1.113},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0509999999999999,"15min":1.099},"min":{"1min":0.70499999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.2749999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.0940000000000001}]}]},{"osd":2,"up_from":234,"seq":1005022347499,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328008,"kb_used_data":8256,"kb_used_omap":13,"kb_used_meta":319730,"kb_avail":93417144,"statfs":{"total":95995035648,"available":95659155456,"internally_reserved":0,"allocated":8454144,"data_stored":7051275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13441,"internal_metadata":327404415},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.65500000000000003,"15min":3.0920000000000001},"min":{"1min":0.35799999999999998,"5min":0.23799999999999999,"15min":0.186},"max":{"1min":1.4370000000000001,"5min":1.4370000000000001,"15min":283.30799999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.71099999999999997,"15min":3.4119999999999999},"min":{"1min":0.41999999999999998,"5min":0.33800000000000002,"15min":0.17999999999999999},"max":{"1min":1.8560000000000001,"5min":1.8560000000000001,"15min":282.24099999999999},"last":0.78600000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.76500000000000001,"15min":4.1239999999999997},"min":{"1min":0.51200000000000001,"5min":0.32700000000000001,"15min":0.218},"max":{"1min":1.8149999999999999,"5min":1.8149999999999999,"15min":267.54000000000002},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.746,"15min":4.2039999999999997},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.22900000000000001},"max":{"1min":1.833,"5min":1.833,"15min":268.12200000000001},"last":0.83199999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.77300000000000002,"15min":1.371},"min":{"1min":0.60899999999999999,"5min":0.309,"15min":0.23100000000000001},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":14.334},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.751,"15min":1.331},"min":{"1min":0.35399999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":1.2569999999999999,"5min":1.327,"15min":13.167999999999999},"last":1.137}]},{"osd":4,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.95199999999999996,"15min":1.254},"min":{"1min":0.63600000000000001,"5min":0.59599999999999997,"15min":0.33000000000000002},"max":{"1min":1.655,"5min":1.655,"15min":10.051},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.99299999999999999,"15min":1.2490000000000001},"min":{"1min":0.57499999999999996,"5min":0.54900000000000004,"15min":0.316},"max":{"1min":1.7549999999999999,"5min":1.7549999999999999,"15min":14.063000000000001},"last":1.1850000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.008,"15min":1.266},"min":{"1min":0.76400000000000001,"5min":0.56299999999999994,"15min":0.32800000000000001},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":10.856999999999999},"last":1.2709999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.014,"15min":1.3999999999999999},"min":{"1min":0.73699999999999999,"5min":0.50600000000000001,"15min":0.316},"max":{"1min":1.79,"5min":1.79,"15min":11.75},"last":1.6619999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.046,"15min":1.4630000000000001},"min":{"1min":0.70099999999999996,"5min":0.51600000000000001,"15min":0.34999999999999998},"max":{"1min":1.677,"5min":1.677,"15min":14.012},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.069,"15min":1.4279999999999999},"min":{"1min":0.82999999999999996,"5min":0.60299999999999998,"15min":0.34699999999999998},"max":{"1min":1.6679999999999999,"5min":1.679,"15min":10.890000000000001},"last":1.2070000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:03:54 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.115,"15min":1.1220000000000001},"min":{"1min":0.73299999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.355,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":1.5720000000000001},{"interface":"front","average":{"1min":1.149,"5min":1.157,"15min":1.1519999999999999},"min":{"1min":0.70399999999999996,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":3.915,"5min":3.915,"15min":3.915},"last":1.246}]}]},{"osd":3,"up_from":103,"seq":442381631974,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917984,"kb_used_data":4184,"kb_used_omap":5,"kb_used_meta":913786,"kb_avail":92827168,"statfs":{"total":95995035648,"available":95055020032,"internally_reserved":0,"allocated":4284416,"data_stored":2863792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5970,"internal_metadata":935717038},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.67700000000000005,"15min":3.6699999999999999},"min":{"1min":0.46700000000000003,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":1.1100000000000001,"5min":1.236,"15min":263.22199999999998},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.72799999999999998,"15min":3.9340000000000002},"min":{"1min":0.498,"5min":0.32600000000000001,"15min":0.23200000000000001},"max":{"1min":1.089,"5min":1.4259999999999999,"15min":261.74599999999998},"last":0.95099999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.66100000000000003,"15min":2.101},"min":{"1min":0.42999999999999999,"5min":0.27300000000000002,"15min":0.21199999999999999},"max":{"1min":0.97499999999999998,"5min":1.141,"15min":143.34100000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.70399999999999996,"15min":2.3370000000000002},"min":{"1min":0.41099999999999998,"5min":0.29199999999999998,"15min":0.20399999999999999},"max":{"1min":1.137,"5min":1.202,"15min":149.15700000000001},"last":0.96999999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.74199999999999999,"15min":1.202},"min":{"1min":0.42599999999999999,"5min":0.36499999999999999,"15min":0.25},"max":{"1min":1.0509999999999999,"5min":1.4410000000000001,"15min":16.620000000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.73399999999999999,"15min":1.613},"min":{"1min":0.44,"5min":0.33900000000000002,"15min":0.20899999999999999},"max":{"1min":1.1319999999999999,"5min":1.3089999999999999,"15min":16.649000000000001},"last":0.85699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.93000000000000005,"15min":1.226},"min":{"1min":0.65200000000000002,"5min":0.499,"15min":0.32100000000000001},"max":{"1min":1.258,"5min":1.3759999999999999,"15min":8.9930000000000003},"last":1.5169999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.97299999999999998,"15min":1.4139999999999999},"min":{"1min":0.64300000000000002,"5min":0.58899999999999997,"15min":0.32000000000000001},"max":{"1min":1.304,"5min":1.546,"15min":15.522},"last":1.1040000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.99199999999999999,"15min":1.8080000000000001},"min":{"1min":0.65500000000000003,"5min":0.56000000000000005,"15min":0.34799999999999998},"max":{"1min":1.28,"5min":1.825,"15min":23.073},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.024,"15min":1.7290000000000001},"min":{"1min":0.72799999999999998,"5min":0.61099999999999999,"15min":0.379},"max":{"1min":1.3620000000000001,"5min":1.673,"15min":22.832000000000001},"last":1.2749999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98799999999999999,"15min":1.409},"min":{"1min":0.67000000000000004,"5min":0.52500000000000002,"15min":0.32700000000000001},"max":{"1min":1.468,"5min":1.712,"15min":10.102},"last":1.323},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0289999999999999,"15min":1.3440000000000001},"min":{"1min":0.78700000000000003,"5min":0.49399999999999999,"15min":0.35999999999999999},"max":{"1min":1.3979999999999999,"5min":1.631,"15min":8.907},"last":1.468}]},{"osd":7,"last update":"Mon Oct 9 05:03:58 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.1180000000000001,"15min":1.1519999999999999},"min":{"1min":0.78200000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.222,"5min":1.823,"15min":1.823},"last":1.3799999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1319999999999999,"15min":1.1699999999999999},"min":{"1min":0.69699999999999995,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.2370000000000001,"5min":1.772,"15min":1.772},"last":1.4950000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019576,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902012,"kb_used_data":7412,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":92843140,"statfs":{"total":95995035648,"available":95071375360,"internally_reserved":0,"allocated":7589888,"data_stored":6950418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.84899999999999998,"15min":1.4199999999999999},"min":{"1min":0.59399999999999997,"5min":0.48999999999999999,"15min":0.26400000000000001},"max":{"1min":1.23,"5min":1.23,"15min":51.948999999999998},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.89000000000000001,"15min":1.341},"min":{"1min":0.62,"5min":0.46200000000000002,"15min":0.26300000000000001},"max":{"1min":1.1659999999999999,"5min":1.603,"15min":52.222000000000001},"last":0.83899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.90000000000000002,"15min":1.9530000000000001},"min":{"1min":0.623,"5min":0.47199999999999998,"15min":0.27700000000000002},"max":{"1min":1.218,"5min":1.409,"15min":81.114999999999995},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.97399999999999998,"15min":2.056},"min":{"1min":0.61399999999999999,"5min":0.57899999999999996,"15min":0.27600000000000002},"max":{"1min":1.218,"5min":1.456,"15min":80.819999999999993},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.88900000000000001,"15min":1.0680000000000001},"min":{"1min":0.55000000000000004,"5min":0.41899999999999998,"15min":0.29799999999999999},"max":{"1min":1.254,"5min":1.3089999999999999,"15min":8.6310000000000002},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.079,"5min":0.99099999999999999,"15min":1.242},"min":{"1min":0.67200000000000004,"5min":0.54600000000000004,"15min":0.28599999999999998},"max":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":15.34},"last":0.89800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.96599999999999997,"15min":1.266},"min":{"1min":0.69599999999999995,"5min":0.57499999999999996,"15min":0.29699999999999999},"max":{"1min":1.2769999999999999,"5min":1.5149999999999999,"15min":32.362000000000002},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.036,"15min":1.478},"min":{"1min":0.79400000000000004,"5min":0.47999999999999998,"15min":0.311},"max":{"1min":1.3240000000000001,"5min":1.365,"15min":32.387},"last":0.98699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.98199999999999998,"15min":1.5640000000000001},"min":{"1min":0.63300000000000001,"5min":0.46500000000000002,"15min":0.311},"max":{"1min":1.47,"5min":1.47,"15min":78.885000000000005},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.93700000000000006,"15min":1.3500000000000001},"min":{"1min":0.73599999999999999,"5min":0.40000000000000002,"15min":0.307},"max":{"1min":1.2190000000000001,"5min":1.349,"15min":78.472999999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.997,"15min":1.0129999999999999},"min":{"1min":0.56399999999999995,"5min":0.53500000000000003,"15min":0.316},"max":{"1min":1.333,"5min":1.452,"15min":8.6999999999999993},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.087,"5min":1.044,"15min":1.345},"min":{"1min":0.63800000000000001,"5min":0.502,"15min":0.30399999999999999},"max":{"1min":1.417,"5min":1.5609999999999999,"15min":9.3759999999999994},"last":0.93700000000000006}]},{"osd":7,"last update":"Mon Oct 9 05:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.107,"15min":1.038},"min":{"1min":0.82499999999999996,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.3720000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.149,"15min":1.075},"min":{"1min":0.82499999999999996,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.7110000000000001,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":0.49299999999999999}]}]},{"osd":5,"up_from":150,"seq":644245094758,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963244,"kb_used_data":7396,"kb_used_omap":6,"kb_used_meta":955833,"kb_avail":92781908,"statfs":{"total":95995035648,"available":95008673792,"internally_reserved":0,"allocated":7573504,"data_stored":6945866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":978773267},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.92400000000000004,"15min":3.7999999999999998},"min":{"1min":0.623,"5min":0.374,"15min":0.26500000000000001},"max":{"1min":1.6850000000000001,"5min":2.3399999999999999,"15min":363.64499999999998},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.92900000000000005,"15min":3.8929999999999998},"min":{"1min":0.53200000000000003,"5min":0.50700000000000001,"15min":0.26600000000000001},"max":{"1min":1.3999999999999999,"5min":1.452,"15min":363.26900000000001},"last":0.77200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.88,"15min":2.6179999999999999},"min":{"1min":0.621,"5min":0.47399999999999998,"15min":0.308},"max":{"1min":1.333,"5min":1.3959999999999999,"15min":130.762},"last":1.026},{"interface":"front","average":{"1min":1.159,"5min":0.97199999999999998,"15min":2.847},"min":{"1min":0.69299999999999995,"5min":0.53300000000000003,"15min":0.29499999999999998},"max":{"1min":4.6559999999999997,"5min":4.6559999999999997,"15min":130.86199999999999},"last":1.0760000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.93000000000000005,"15min":1.712},"min":{"1min":0.624,"5min":0.52100000000000002,"15min":0.28399999999999997},"max":{"1min":1.5860000000000001,"5min":1.5860000000000001,"15min":208.40100000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.94399999999999995,"15min":1.9490000000000001},"min":{"1min":0.73799999999999999,"5min":0.55000000000000004,"15min":0.29899999999999999},"max":{"1min":1.1919999999999999,"5min":1.4890000000000001,"15min":208.078},"last":0.91400000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.006,"15min":2.153},"min":{"1min":0.72499999999999998,"5min":0.625,"15min":0.31900000000000001},"max":{"1min":4.7729999999999997,"5min":4.7729999999999997,"15min":130.69200000000001},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.216,"5min":1.0680000000000001,"15min":2.2949999999999999},"min":{"1min":0.64300000000000002,"5min":0.624,"15min":0.33000000000000002},"max":{"1min":4.734,"5min":4.734,"15min":130.53399999999999},"last":1.165}]},{"osd":4,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.048,"15min":1.208},"min":{"1min":0.74199999999999999,"5min":0.44500000000000001,"15min":0.34300000000000003},"max":{"1min":4.8129999999999997,"5min":4.8129999999999997,"15min":12.734},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.056,"15min":1.135},"min":{"1min":0.68000000000000005,"5min":0.42299999999999999,"15min":0.28100000000000003},"max":{"1min":4.8010000000000002,"5min":4.8010000000000002,"15min":8.3409999999999993},"last":1.0509999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":1.103,"15min":1.0900000000000001},"min":{"1min":0.76900000000000002,"5min":0.377,"15min":0.33900000000000002},"max":{"1min":4.7599999999999998,"5min":4.7599999999999998,"15min":7.2489999999999997},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.319,"5min":1.1279999999999999,"15min":1.3500000000000001},"min":{"1min":0.79600000000000004,"5min":0.44800000000000001,"15min":0.28499999999999998},"max":{"1min":4.8440000000000003,"5min":4.8440000000000003,"15min":14.223000000000001},"last":1.1160000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.1879999999999999,"15min":1.1339999999999999},"min":{"1min":0.84199999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":4.8789999999999996,"5min":4.8789999999999996,"15min":4.8789999999999996},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.391,"5min":1.236,"15min":1.2010000000000001},"min":{"1min":0.84599999999999997,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":1.107}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4219,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2197,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2068,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9743,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9222,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3773,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6893,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5536,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11569,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:04:20.664 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:04:20.664 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:04:20.739 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=55dd3cd278a540a89713f79d9b321ad9 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:04:20.741 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T05:04:20.762 INFO:tasks.cephadm:Teardown begin 2023-10-09T05:04:20.763 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:04:20.792 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:04:20.808 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T05:04:20.808 DEBUG:teuthology.orchestra.run.smithi082:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:04:20.845 DEBUG:teuthology.orchestra.run.smithi153:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:04:20.862 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T05:04:20.862 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T05:04:20.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T05:04:21.200 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:20 smithi082 systemd[1]: Stopping Ceph mon.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:21.201 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[77063]: debug 2023-10-09T05:04:21.065+0000 7f3fcf0d1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:04:21.201 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[77063]: debug 2023-10-09T05:04:21.065+0000 7f3fcf0d1700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T05:04:21.201 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE Bus STOPPING 2023-10-09T05:04:21.201 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:04:21.202 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE Bus STOPPED 2023-10-09T05:04:21.202 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE Bus STARTING 2023-10-09T05:04:21.202 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE Serving on http://:::9283 2023-10-09T05:04:21.202 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:21 smithi082 bash[73210]: [09/Oct/2023:05:04:21] ENGINE Bus STARTED 2023-10-09T05:04:22.297 INFO:journalctl@ceph.mon.a.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[260947]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T05:04:22.320 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T05:04:22.430 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:22.430 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T05:04:22.430 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T05:04:22.431 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T05:04:22.639 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:22 smithi082 systemd[1]: Stopping Ceph mon.c for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:22.959 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[79733]: debug 2023-10-09T05:04:22.636+0000 7f86e3676700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:04:22.960 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[79733]: debug 2023-10-09T05:04:22.636+0000 7f86e3676700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:04:22.960 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE Bus STOPPING 2023-10-09T05:04:22.960 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:04:22.960 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE Bus STOPPED 2023-10-09T05:04:22.960 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE Bus STARTING 2023-10-09T05:04:22.961 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE Serving on http://:::9283 2023-10-09T05:04:22.961 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:22 smithi082 bash[73210]: [09/Oct/2023:05:04:22] ENGINE Bus STARTED 2023-10-09T05:04:23.765 INFO:journalctl@ceph.mon.c.smithi082.stdout:Oct 09 05:04:23 smithi082 bash[261084]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T05:04:23.778 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T05:04:23.847 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:23.847 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T05:04:23.848 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T05:04:23.848 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T05:04:24.146 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:23 smithi153 systemd[1]: Stopping Ceph mon.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:24.402 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:24 smithi153 bash[59565]: debug 2023-10-09T05:04:24.140+0000 7fb986fd3700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:04:24.403 INFO:journalctl@ceph.mon.b.smithi153.stdout:Oct 09 05:04:24 smithi153 bash[59565]: debug 2023-10-09T05:04:24.140+0000 7fb986fd3700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:04:25.476 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T05:04:25.511 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:25.511 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T05:04:25.512 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T05:04:25.512 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T05:04:25.792 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:25 smithi082 systemd[1]: Stopping Ceph mgr.y for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:25.792 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:25 smithi082 bash[73210]: debug 2023-10-09T05:04:25.704+0000 7fed4e8bb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:04:25.793 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:25 smithi082 bash[73210]: debug 2023-10-09T05:04:25.704+0000 7fed4e8bb700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T05:04:26.715 INFO:journalctl@ceph.mgr.y.smithi082.stdout:Oct 09 05:04:26 smithi082 bash[261196]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T05:04:26.722 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T05:04:26.791 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:26.791 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T05:04:26.791 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T05:04:26.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T05:04:27.065 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:26 smithi153 systemd[1]: Stopping Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:27.956 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:27 smithi153 bash[158146]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T05:04:27.957 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:27 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:04:27.957 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:27 smithi153 bash[158208]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T05:04:27.957 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:27 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T05:04:27.957 INFO:journalctl@ceph.mgr.x.smithi153.stdout:Oct 09 05:04:27 smithi153 systemd[1]: Stopped Ceph mgr.x for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:04:27.958 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T05:04:27.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:27.987 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T05:04:27.988 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T05:04:27.988 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T05:04:28.524 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:28 smithi082 systemd[1]: Stopping Ceph osd.0 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:28.524 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:28 smithi082 bash[85611]: debug 2023-10-09T05:04:28.164+0000 7fddf621e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:04:28.524 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:28 smithi082 bash[85611]: debug 2023-10-09T05:04:28.164+0000 7fddf621e700 -1 osd.0 382 *** Got signal Terminated *** 2023-10-09T05:04:28.524 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:28 smithi082 bash[85611]: debug 2023-10-09T05:04:28.164+0000 7fddf621e700 -1 osd.0 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:04:34.871 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:34 smithi082 bash[261297]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T05:04:34.872 INFO:journalctl@ceph.osd.0.smithi082.stdout:Oct 09 05:04:34 smithi082 bash[261365]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T05:04:37.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T05:04:37.566 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:37.566 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T05:04:37.566 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T05:04:37.567 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T05:04:37.731 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:37 smithi082 systemd[1]: Stopping Ceph osd.1 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:38.231 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:37 smithi082 bash[88365]: debug 2023-10-09T05:04:37.756+0000 7f9ffbafb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:04:38.231 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:37 smithi082 bash[88365]: debug 2023-10-09T05:04:37.756+0000 7f9ffbafb700 -1 osd.1 382 *** Got signal Terminated *** 2023-10-09T05:04:38.231 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:37 smithi082 bash[88365]: debug 2023-10-09T05:04:37.756+0000 7f9ffbafb700 -1 osd.1 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:04:44.481 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:44 smithi082 bash[261510]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T05:04:44.481 INFO:journalctl@ceph.osd.1.smithi082.stdout:Oct 09 05:04:44 smithi082 bash[261579]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T05:04:46.459 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T05:04:46.498 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:46.498 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T05:04:46.499 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T05:04:46.499 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T05:04:46.709 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:46 smithi082 systemd[1]: Stopping Ceph osd.2 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:46.981 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:46 smithi082 bash[183703]: debug 2023-10-09T05:04:46.704+0000 7f6901242700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:04:46.981 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:46 smithi082 bash[183703]: debug 2023-10-09T05:04:46.704+0000 7f6901242700 -1 osd.2 382 *** Got signal Terminated *** 2023-10-09T05:04:46.981 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:46 smithi082 bash[183703]: debug 2023-10-09T05:04:46.704+0000 7f6901242700 -1 osd.2 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:04:52.175 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:04:51 smithi153 bash[55216]: ts=2023-10-09T05:04:51.679Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.82:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.82:8765: connect: connection refused" 2023-10-09T05:04:52.175 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:04:51 smithi153 bash[55216]: ts=2023-10-09T05:04:51.680Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.82:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.82:8765: connect: connection refused" 2023-10-09T05:04:52.175 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:04:51 smithi153 bash[55216]: ts=2023-10-09T05:04:51.682Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.82:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.82:8765: connect: connection refused" 2023-10-09T05:04:52.175 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:04:51 smithi153 bash[55216]: ts=2023-10-09T05:04:51.684Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.82:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.82:8765: connect: connection refused" 2023-10-09T05:04:53.188 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:52 smithi082 bash[261717]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:04:53.188 INFO:journalctl@ceph.osd.2.smithi082.stdout:Oct 09 05:04:53 smithi082 bash[261783]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T05:04:55.174 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:55 smithi153 bash[153020]: debug 2023-10-09T05:04:55.028+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:04:55.175 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:54 smithi153 bash[31429]: debug 2023-10-09T05:04:54.896+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:04:55.292 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T05:04:55.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:04:55.324 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T05:04:55.325 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T05:04:55.325 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T05:04:55.551 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:55 smithi082 systemd[1]: Stopping Ceph osd.3 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:04:55.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:55 smithi082 bash[94164]: debug 2023-10-09T05:04:55.548+0000 7f295550a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:04:55.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:55 smithi082 bash[94164]: debug 2023-10-09T05:04:55.548+0000 7f295550a700 -1 osd.3 382 *** Got signal Terminated *** 2023-10-09T05:04:55.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:55 smithi082 bash[94164]: debug 2023-10-09T05:04:55.548+0000 7f295550a700 -1 osd.3 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:04:56.424 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:55 smithi153 bash[31429]: debug 2023-10-09T05:04:55.936+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:04:56.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:55 smithi153 bash[153020]: debug 2023-10-09T05:04:55.988+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:04:56.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:04:56 smithi153 bash[83720]: debug 2023-10-09T05:04:56.032+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:04:57.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:57 smithi153 bash[153020]: debug 2023-10-09T05:04:57.008+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:04:57.425 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:56 smithi153 bash[31429]: debug 2023-10-09T05:04:56.944+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:04:57.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:04:56 smithi153 bash[81466]: debug 2023-10-09T05:04:56.980+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:04:57.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:04:57 smithi153 bash[83720]: debug 2023-10-09T05:04:57.060+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:04:57.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:57 smithi082 bash[94164]: debug 2023-10-09T05:04:57.676+0000 7f294e2f9700 -1 osd.3 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.951460+0000 front 2023-10-09T05:04:30.951573+0000 (oldest deadline 2023-10-09T05:04:56.851148+0000) 2023-10-09T05:04:58.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:04:57 smithi153 bash[81466]: debug 2023-10-09T05:04:57.988+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:04:58.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:04:58 smithi153 bash[83720]: debug 2023-10-09T05:04:58.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:04:58.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:58 smithi153 bash[153020]: debug 2023-10-09T05:04:58.028+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:04:58.426 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:57 smithi153 bash[31429]: debug 2023-10-09T05:04:57.932+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:04:58.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:58 smithi082 bash[94164]: debug 2023-10-09T05:04:58.672+0000 7f294e2f9700 -1 osd.3 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.951460+0000 front 2023-10-09T05:04:30.951573+0000 (oldest deadline 2023-10-09T05:04:56.851148+0000) 2023-10-09T05:04:59.174 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:59 smithi153 bash[153020]: debug 2023-10-09T05:04:59.004+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:04:59.175 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:58 smithi153 bash[31429]: debug 2023-10-09T05:04:58.884+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:04:59.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:04:59 smithi153 bash[81466]: debug 2023-10-09T05:04:59.008+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:04:59.176 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:04:59 smithi153 bash[83720]: debug 2023-10-09T05:04:59.120+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:04:59.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:04:59 smithi082 bash[94164]: debug 2023-10-09T05:04:59.668+0000 7f294e2f9700 -1 osd.3 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.951460+0000 front 2023-10-09T05:04:30.951573+0000 (oldest deadline 2023-10-09T05:04:56.851148+0000) 2023-10-09T05:05:00.174 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:04:59 smithi153 bash[31429]: debug 2023-10-09T05:04:59.888+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:00.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:00 smithi153 bash[81466]: debug 2023-10-09T05:04:59.996+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:00.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:00 smithi153 bash[83720]: debug 2023-10-09T05:05:00.132+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:00.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:04:59 smithi153 bash[153020]: debug 2023-10-09T05:04:59.976+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:01.174 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:00 smithi153 bash[31429]: debug 2023-10-09T05:05:00.876+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:01.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:01 smithi153 bash[81466]: debug 2023-10-09T05:05:01.020+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:01.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:01 smithi153 bash[83720]: debug 2023-10-09T05:05:01.092+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:01.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:00 smithi153 bash[153020]: debug 2023-10-09T05:05:00.976+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:01.981 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:05:01 smithi082 bash[261925]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:05:02.174 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:01 smithi153 bash[31429]: debug 2023-10-09T05:05:01.912+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:02.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:02 smithi153 bash[81466]: debug 2023-10-09T05:05:02.020+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:02.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:02 smithi153 bash[83720]: debug 2023-10-09T05:05:02.132+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:02.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:01 smithi153 bash[153020]: debug 2023-10-09T05:05:01.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:02.481 INFO:journalctl@ceph.osd.3.smithi082.stdout:Oct 09 05:05:02 smithi082 bash[261994]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T05:05:03.144 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:03 smithi153 bash[81466]: debug 2023-10-09T05:05:03.040+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:03.145 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:02 smithi153 bash[153020]: debug 2023-10-09T05:05:02.964+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:03.145 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:02 smithi153 bash[31429]: debug 2023-10-09T05:05:02.876+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:03.424 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:03 smithi153 bash[83720]: debug 2023-10-09T05:05:03.140+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:04.169 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[81466]: debug 2023-10-09T05:05:04.064+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:04.170 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:03 smithi153 bash[153020]: debug 2023-10-09T05:05:03.916+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:04.170 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:03 smithi153 bash[31429]: debug 2023-10-09T05:05:03.832+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:04.170 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:03 smithi153 bash[31429]: debug 2023-10-09T05:05:03.832+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:04.206 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T05:05:04.238 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:04.239 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T05:05:04.239 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T05:05:04.239 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T05:05:04.424 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[83720]: debug 2023-10-09T05:05:04.168+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:04.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[83720]: debug 2023-10-09T05:05:04.168+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:04.425 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 systemd[1]: Stopping Ceph osd.4 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:04.854 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[31429]: debug 2023-10-09T05:05:04.420+0000 7f74dfff1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:05:04.854 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[31429]: debug 2023-10-09T05:05:04.420+0000 7f74dfff1700 -1 osd.4 382 *** Got signal Terminated *** 2023-10-09T05:05:04.854 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[31429]: debug 2023-10-09T05:05:04.420+0000 7f74dfff1700 -1 osd.4 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:05:05.158 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[81466]: debug 2023-10-09T05:05:05.084+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:05.158 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[31429]: debug 2023-10-09T05:05:04.848+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:05.158 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[31429]: debug 2023-10-09T05:05:04.848+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:05.159 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[153020]: debug 2023-10-09T05:05:04.948+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:05.159 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:04 smithi153 bash[153020]: debug 2023-10-09T05:05:04.948+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:05.424 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[83720]: debug 2023-10-09T05:05:05.152+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:05.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[83720]: debug 2023-10-09T05:05:05.152+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:06.124 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[153020]: debug 2023-10-09T05:05:05.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:06.124 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[153020]: debug 2023-10-09T05:05:05.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:06.125 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[31429]: debug 2023-10-09T05:05:05.848+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:06.125 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:05 smithi153 bash[31429]: debug 2023-10-09T05:05:05.848+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:06.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[81466]: debug 2023-10-09T05:05:06.120+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:06.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[83720]: debug 2023-10-09T05:05:06.124+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:06.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[83720]: debug 2023-10-09T05:05:06.124+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:07.131 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[31429]: debug 2023-10-09T05:05:06.828+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:07.131 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[31429]: debug 2023-10-09T05:05:06.828+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:07.132 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[153020]: debug 2023-10-09T05:05:06.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:07.132 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:06 smithi153 bash[153020]: debug 2023-10-09T05:05:06.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:07.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[81466]: debug 2023-10-09T05:05:07.132+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:07.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[83720]: debug 2023-10-09T05:05:07.124+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:07.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[83720]: debug 2023-10-09T05:05:07.124+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:08.150 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[153020]: debug 2023-10-09T05:05:07.968+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:08.151 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[153020]: debug 2023-10-09T05:05:07.968+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:08.151 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[31429]: debug 2023-10-09T05:05:07.844+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:08.151 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:07 smithi153 bash[31429]: debug 2023-10-09T05:05:07.844+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:08.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[81466]: debug 2023-10-09T05:05:08.148+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:08.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[81466]: debug 2023-10-09T05:05:08.148+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:08.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[83720]: debug 2023-10-09T05:05:08.144+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:08.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[83720]: debug 2023-10-09T05:05:08.144+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:09.135 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[83720]: debug 2023-10-09T05:05:09.108+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:09.135 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[83720]: debug 2023-10-09T05:05:09.108+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:09.135 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[153020]: debug 2023-10-09T05:05:08.932+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:09.136 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[153020]: debug 2023-10-09T05:05:08.932+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:09.136 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[31429]: debug 2023-10-09T05:05:08.872+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:30.019281+0000 front 2023-10-09T05:04:30.018958+0000 (oldest deadline 2023-10-09T05:04:54.118833+0000) 2023-10-09T05:05:09.136 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:08 smithi153 bash[31429]: debug 2023-10-09T05:05:08.872+0000 7f74db76b700 -1 osd.4 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:37.520915+0000 front 2023-10-09T05:04:37.521128+0000 (oldest deadline 2023-10-09T05:05:03.420554+0000) 2023-10-09T05:05:09.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[81466]: debug 2023-10-09T05:05:09.132+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:09.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[81466]: debug 2023-10-09T05:05:09.132+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:10.246 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[83720]: debug 2023-10-09T05:05:10.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:10.246 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[83720]: debug 2023-10-09T05:05:10.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:10.246 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[153020]: debug 2023-10-09T05:05:09.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:10.247 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:09 smithi153 bash[153020]: debug 2023-10-09T05:05:09.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:10.674 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[158250]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T05:05:10.675 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[81466]: debug 2023-10-09T05:05:10.148+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:10.675 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[81466]: debug 2023-10-09T05:05:10.148+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:10.960 INFO:journalctl@ceph.osd.4.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[158315]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T05:05:11.103 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[153020]: debug 2023-10-09T05:05:10.952+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:11.103 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:10 smithi153 bash[153020]: debug 2023-10-09T05:05:10.952+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:11.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[81466]: debug 2023-10-09T05:05:11.116+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:11.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[81466]: debug 2023-10-09T05:05:11.116+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:11.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[83720]: debug 2023-10-09T05:05:11.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:11.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[83720]: debug 2023-10-09T05:05:11.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:12.097 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[153020]: debug 2023-10-09T05:05:11.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:12.098 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:11 smithi153 bash[153020]: debug 2023-10-09T05:05:11.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:12.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[81466]: debug 2023-10-09T05:05:12.100+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:12.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[81466]: debug 2023-10-09T05:05:12.100+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:12.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[83720]: debug 2023-10-09T05:05:12.092+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:12.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[83720]: debug 2023-10-09T05:05:12.092+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:13.007 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T05:05:13.039 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:13.039 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T05:05:13.039 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T05:05:13.040 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T05:05:13.269 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[81466]: debug 2023-10-09T05:05:13.072+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:13.269 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[81466]: debug 2023-10-09T05:05:13.072+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:13.270 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 systemd[1]: Stopping Ceph osd.5 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:13.270 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[83720]: debug 2023-10-09T05:05:13.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:13.270 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[83720]: debug 2023-10-09T05:05:13.096+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:13.270 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[153020]: debug 2023-10-09T05:05:12.924+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:13.271 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[153020]: debug 2023-10-09T05:05:12.924+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:13.271 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:12 smithi153 bash[153020]: debug 2023-10-09T05:05:12.924+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:13.674 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[83720]: debug 2023-10-09T05:05:13.264+0000 7f3747f86700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:05:13.675 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[83720]: debug 2023-10-09T05:05:13.264+0000 7f3747f86700 -1 osd.5 382 *** Got signal Terminated *** 2023-10-09T05:05:13.675 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[83720]: debug 2023-10-09T05:05:13.264+0000 7f3747f86700 -1 osd.5 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:05:14.174 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[81466]: debug 2023-10-09T05:05:14.116+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:14.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[81466]: debug 2023-10-09T05:05:14.116+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:14.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[83720]: debug 2023-10-09T05:05:14.108+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:14.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[83720]: debug 2023-10-09T05:05:14.108+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:14.175 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[153020]: debug 2023-10-09T05:05:13.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:14.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[153020]: debug 2023-10-09T05:05:13.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:14.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:13 smithi153 bash[153020]: debug 2023-10-09T05:05:13.912+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:15.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[81466]: debug 2023-10-09T05:05:15.076+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:15.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[81466]: debug 2023-10-09T05:05:15.076+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:15.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[81466]: debug 2023-10-09T05:05:15.076+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:15.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[83720]: debug 2023-10-09T05:05:15.112+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:15.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[83720]: debug 2023-10-09T05:05:15.112+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:15.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[83720]: debug 2023-10-09T05:05:15.112+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:50.917199+0000 front 2023-10-09T05:04:50.917218+0000 (oldest deadline 2023-10-09T05:05:15.016699+0000) 2023-10-09T05:05:15.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[153020]: debug 2023-10-09T05:05:14.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:15.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[153020]: debug 2023-10-09T05:05:14.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:15.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:14 smithi153 bash[153020]: debug 2023-10-09T05:05:14.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:16.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[81466]: debug 2023-10-09T05:05:16.032+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:16.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[81466]: debug 2023-10-09T05:05:16.032+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:16.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[81466]: debug 2023-10-09T05:05:16.032+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:16.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[83720]: debug 2023-10-09T05:05:16.156+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:16.426 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[83720]: debug 2023-10-09T05:05:16.156+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:16.426 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[83720]: debug 2023-10-09T05:05:16.156+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:50.917199+0000 front 2023-10-09T05:04:50.917218+0000 (oldest deadline 2023-10-09T05:05:15.016699+0000) 2023-10-09T05:05:16.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[153020]: debug 2023-10-09T05:05:15.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:16.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[153020]: debug 2023-10-09T05:05:15.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:16.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:15 smithi153 bash[153020]: debug 2023-10-09T05:05:15.956+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:17.174 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[83720]: debug 2023-10-09T05:05:17.132+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:17.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[83720]: debug 2023-10-09T05:05:17.132+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:17.175 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[83720]: debug 2023-10-09T05:05:17.132+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:50.917199+0000 front 2023-10-09T05:04:50.917218+0000 (oldest deadline 2023-10-09T05:05:15.016699+0000) 2023-10-09T05:05:17.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[81466]: debug 2023-10-09T05:05:17.028+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:17.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[81466]: debug 2023-10-09T05:05:17.028+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:17.175 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[81466]: debug 2023-10-09T05:05:17.028+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:17.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[153020]: debug 2023-10-09T05:05:16.920+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:17.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[153020]: debug 2023-10-09T05:05:16.920+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:17.176 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:16 smithi153 bash[153020]: debug 2023-10-09T05:05:16.920+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:18.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[83720]: debug 2023-10-09T05:05:18.136+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.912610+0000 front 2023-10-09T05:04:31.912557+0000 (oldest deadline 2023-10-09T05:04:56.012104+0000) 2023-10-09T05:05:18.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[83720]: debug 2023-10-09T05:05:18.136+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.213998+0000 front 2023-10-09T05:04:41.214266+0000 (oldest deadline 2023-10-09T05:05:03.513638+0000) 2023-10-09T05:05:18.425 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[83720]: debug 2023-10-09T05:05:18.136+0000 7f3745704700 -1 osd.5 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:50.917199+0000 front 2023-10-09T05:04:50.917218+0000 (oldest deadline 2023-10-09T05:05:15.016699+0000) 2023-10-09T05:05:18.426 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[81466]: debug 2023-10-09T05:05:18.060+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:18.426 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[81466]: debug 2023-10-09T05:05:18.060+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:18.427 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[81466]: debug 2023-10-09T05:05:18.060+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:18.427 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[153020]: debug 2023-10-09T05:05:17.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:18.427 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[153020]: debug 2023-10-09T05:05:17.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:18.428 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:17 smithi153 bash[153020]: debug 2023-10-09T05:05:17.940+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:19.307 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:19 smithi153 bash[158470]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T05:05:19.308 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:19 smithi153 bash[81466]: debug 2023-10-09T05:05:19.040+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:19.308 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:19 smithi153 bash[81466]: debug 2023-10-09T05:05:19.040+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:19.308 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:19 smithi153 bash[81466]: debug 2023-10-09T05:05:19.040+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:19.308 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[153020]: debug 2023-10-09T05:05:18.972+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:19.308 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[153020]: debug 2023-10-09T05:05:18.972+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:19.309 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:18 smithi153 bash[153020]: debug 2023-10-09T05:05:18.972+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:19.674 INFO:journalctl@ceph.osd.5.smithi153.stdout:Oct 09 05:05:19 smithi153 bash[158537]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T05:05:20.305 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[153020]: debug 2023-10-09T05:05:20.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:20.306 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[153020]: debug 2023-10-09T05:05:20.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:20.306 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[153020]: debug 2023-10-09T05:05:20.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:20.306 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[81466]: debug 2023-10-09T05:05:20.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:20.306 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[81466]: debug 2023-10-09T05:05:20.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:20.306 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:20 smithi153 bash[81466]: debug 2023-10-09T05:05:20.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:21.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.079+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:21.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.079+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:21.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.079+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:21.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[153020]: debug 2023-10-09T05:05:21.003+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:21.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[153020]: debug 2023-10-09T05:05:21.003+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:21.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[153020]: debug 2023-10-09T05:05:21.003+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:21.474 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T05:05:21.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:21.507 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T05:05:21.507 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T05:05:21.507 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T05:05:21.725 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 systemd[1]: Stopping Ceph osd.6 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:22.014 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.723+0000 7f436991d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:05:22.015 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.723+0000 7f436991d700 -1 osd.6 382 *** Got signal Terminated *** 2023-10-09T05:05:22.015 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:21 smithi153 bash[81466]: debug 2023-10-09T05:05:21.723+0000 7f436991d700 -1 osd.6 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:05:22.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[81466]: debug 2023-10-09T05:05:22.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:22.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[81466]: debug 2023-10-09T05:05:22.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:22.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[81466]: debug 2023-10-09T05:05:22.083+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:22.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:22.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:22.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.011+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:23.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[81466]: debug 2023-10-09T05:05:23.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:23.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[81466]: debug 2023-10-09T05:05:23.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:23.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[81466]: debug 2023-10-09T05:05:23.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:23.426 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[81466]: debug 2023-10-09T05:05:23.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:58.131940+0000 front 2023-10-09T05:04:58.132077+0000 (oldest deadline 2023-10-09T05:05:22.831817+0000) 2023-10-09T05:05:23.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.995+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:23.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.995+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:23.427 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:22 smithi153 bash[153020]: debug 2023-10-09T05:05:22.995+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:24.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[81466]: debug 2023-10-09T05:05:24.015+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:24.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[81466]: debug 2023-10-09T05:05:24.015+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:24.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[81466]: debug 2023-10-09T05:05:24.015+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:24.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[81466]: debug 2023-10-09T05:05:24.015+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:58.131940+0000 front 2023-10-09T05:04:58.132077+0000 (oldest deadline 2023-10-09T05:05:22.831817+0000) 2023-10-09T05:05:24.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[153020]: debug 2023-10-09T05:05:23.947+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:24.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[153020]: debug 2023-10-09T05:05:23.947+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:24.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[153020]: debug 2023-10-09T05:05:23.947+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:24.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:23 smithi153 bash[153020]: debug 2023-10-09T05:05:23.947+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:25.424 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:25 smithi153 bash[81466]: debug 2023-10-09T05:05:25.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:25.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:25 smithi153 bash[81466]: debug 2023-10-09T05:05:25.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:25.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:25 smithi153 bash[81466]: debug 2023-10-09T05:05:25.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:25.425 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:25 smithi153 bash[81466]: debug 2023-10-09T05:05:25.047+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:58.131940+0000 front 2023-10-09T05:04:58.132077+0000 (oldest deadline 2023-10-09T05:05:22.831817+0000) 2023-10-09T05:05:25.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[153020]: debug 2023-10-09T05:05:24.991+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:25.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[153020]: debug 2023-10-09T05:05:24.991+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:25.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[153020]: debug 2023-10-09T05:05:24.991+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:25.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:24 smithi153 bash[153020]: debug 2023-10-09T05:05:24.991+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:26.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[153020]: debug 2023-10-09T05:05:26.023+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:26.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[153020]: debug 2023-10-09T05:05:26.023+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:26.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[153020]: debug 2023-10-09T05:05:26.023+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:26.426 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[153020]: debug 2023-10-09T05:05:26.023+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:26.426 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[81466]: debug 2023-10-09T05:05:26.095+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:33.126656+0000 front 2023-10-09T05:04:33.126868+0000 (oldest deadline 2023-10-09T05:04:56.626483+0000) 2023-10-09T05:05:26.426 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[81466]: debug 2023-10-09T05:05:26.095+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:41.928095+0000 front 2023-10-09T05:04:41.927911+0000 (oldest deadline 2023-10-09T05:05:07.227751+0000) 2023-10-09T05:05:26.427 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[81466]: debug 2023-10-09T05:05:26.095+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:48.830040+0000 front 2023-10-09T05:04:48.830216+0000 (oldest deadline 2023-10-09T05:05:14.729750+0000) 2023-10-09T05:05:26.427 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:26 smithi153 bash[81466]: debug 2023-10-09T05:05:26.095+0000 7f436709b700 -1 osd.6 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:58.131940+0000 front 2023-10-09T05:04:58.132077+0000 (oldest deadline 2023-10-09T05:05:22.831817+0000) 2023-10-09T05:05:27.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[153020]: debug 2023-10-09T05:05:27.027+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:27.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[153020]: debug 2023-10-09T05:05:27.027+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:27.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[153020]: debug 2023-10-09T05:05:27.027+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:27.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[153020]: debug 2023-10-09T05:05:27.027+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:27.911 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[158678]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T05:05:27.911 INFO:journalctl@ceph.osd.6.smithi153.stdout:Oct 09 05:05:27 smithi153 bash[158741]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T05:05:28.174 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:28 smithi153 bash[153020]: debug 2023-10-09T05:05:28.055+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:28.175 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:28 smithi153 bash[153020]: debug 2023-10-09T05:05:28.055+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:28.175 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:28 smithi153 bash[153020]: debug 2023-10-09T05:05:28.055+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:28.175 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:28 smithi153 bash[153020]: debug 2023-10-09T05:05:28.055+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:29.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:29 smithi153 bash[153020]: debug 2023-10-09T05:05:29.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:29.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:29 smithi153 bash[153020]: debug 2023-10-09T05:05:29.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:29.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:29 smithi153 bash[153020]: debug 2023-10-09T05:05:29.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:29.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:29 smithi153 bash[153020]: debug 2023-10-09T05:05:29.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:30.021 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T05:05:30.054 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:30.054 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T05:05:30.054 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T05:05:30.054 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T05:05:30.285 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:30.286 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:30.286 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:30.286 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.095+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:30.286 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 systemd[1]: Stopping Ceph osd.7 for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:30.286 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.283+0000 7f69f30d2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:05:30.287 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.283+0000 7f69f30d2700 -1 osd.7 382 *** Got signal Terminated *** 2023-10-09T05:05:30.287 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:30 smithi153 bash[153020]: debug 2023-10-09T05:05:30.283+0000 7f69f30d2700 -1 osd.7 382 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:05:31.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:31 smithi153 bash[153020]: debug 2023-10-09T05:05:31.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:31.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:31 smithi153 bash[153020]: debug 2023-10-09T05:05:31.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:31.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:31 smithi153 bash[153020]: debug 2023-10-09T05:05:31.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:31.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:31 smithi153 bash[153020]: debug 2023-10-09T05:05:31.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:31.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:31 smithi153 bash[153020]: debug 2023-10-09T05:05:31.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2023-10-09T05:05:06.680668+0000 front 2023-10-09T05:05:06.680892+0000 (oldest deadline 2023-10-09T05:05:30.180459+0000) 2023-10-09T05:05:32.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:32 smithi153 bash[153020]: debug 2023-10-09T05:05:32.103+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:32.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:32 smithi153 bash[153020]: debug 2023-10-09T05:05:32.103+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:32.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:32 smithi153 bash[153020]: debug 2023-10-09T05:05:32.103+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:32.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:32 smithi153 bash[153020]: debug 2023-10-09T05:05:32.103+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:32.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:32 smithi153 bash[153020]: debug 2023-10-09T05:05:32.103+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2023-10-09T05:05:06.680668+0000 front 2023-10-09T05:05:06.680892+0000 (oldest deadline 2023-10-09T05:05:30.180459+0000) 2023-10-09T05:05:33.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:33 smithi153 bash[153020]: debug 2023-10-09T05:05:33.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:33 smithi153 bash[153020]: debug 2023-10-09T05:05:33.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:33 smithi153 bash[153020]: debug 2023-10-09T05:05:33.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:33 smithi153 bash[153020]: debug 2023-10-09T05:05:33.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:33.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:33 smithi153 bash[153020]: debug 2023-10-09T05:05:33.111+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2023-10-09T05:05:06.680668+0000 front 2023-10-09T05:05:06.680892+0000 (oldest deadline 2023-10-09T05:05:30.180459+0000) 2023-10-09T05:05:34.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:34 smithi153 bash[153020]: debug 2023-10-09T05:05:34.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:34.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:34 smithi153 bash[153020]: debug 2023-10-09T05:05:34.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:34.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:34 smithi153 bash[153020]: debug 2023-10-09T05:05:34.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:34.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:34 smithi153 bash[153020]: debug 2023-10-09T05:05:34.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:34.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:34 smithi153 bash[153020]: debug 2023-10-09T05:05:34.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2023-10-09T05:05:06.680668+0000 front 2023-10-09T05:05:06.680892+0000 (oldest deadline 2023-10-09T05:05:30.180459+0000) 2023-10-09T05:05:35.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:35 smithi153 bash[153020]: debug 2023-10-09T05:05:35.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2023-10-09T05:04:31.471456+0000 front 2023-10-09T05:04:31.471276+0000 (oldest deadline 2023-10-09T05:04:54.371116+0000) 2023-10-09T05:05:35.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:35 smithi153 bash[153020]: debug 2023-10-09T05:05:35.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2023-10-09T05:04:39.474042+0000 front 2023-10-09T05:04:39.474210+0000 (oldest deadline 2023-10-09T05:05:04.173639+0000) 2023-10-09T05:05:35.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:35 smithi153 bash[153020]: debug 2023-10-09T05:05:35.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2023-10-09T05:04:47.675432+0000 front 2023-10-09T05:04:47.675269+0000 (oldest deadline 2023-10-09T05:05:12.375040+0000) 2023-10-09T05:05:35.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:35 smithi153 bash[153020]: debug 2023-10-09T05:05:35.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2023-10-09T05:04:56.778621+0000 front 2023-10-09T05:04:56.778650+0000 (oldest deadline 2023-10-09T05:05:23.779098+0000) 2023-10-09T05:05:35.425 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:35 smithi153 bash[153020]: debug 2023-10-09T05:05:35.067+0000 7f69f0850700 -1 osd.7 382 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2023-10-09T05:05:06.680668+0000 front 2023-10-09T05:05:06.680892+0000 (oldest deadline 2023-10-09T05:05:30.180459+0000) 2023-10-09T05:05:36.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:36 smithi153 bash[158882]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T05:05:36.424 INFO:journalctl@ceph.osd.7.smithi153.stdout:Oct 09 05:05:36 smithi153 bash[158947]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T05:05:38.198 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T05:05:38.227 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:38.228 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T05:05:38.228 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T05:05:38.228 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T05:05:38.449 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 systemd[1]: Stopping Ceph prometheus.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T05:05:38.450 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:05:38.451 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T05:05:38.451 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.420Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:05:38.451 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.423Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:05:38.451 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.423Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T05:05:38.451 INFO:journalctl@ceph.prometheus.a.smithi153.stdout:Oct 09 05:05:38 smithi153 bash[55216]: ts=2023-10-09T05:05:38.423Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T05:05:39.279 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T05:05:39.309 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:05:39.309 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T05:05:39.309 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:05:41.981 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:41 smithi082 bash[72338]: level=warn ts=2023-10-09T05:05:41.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi082.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.82:8443: connect: connection refused" 2023-10-09T05:05:41.981 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:41 smithi082 bash[72338]: level=warn ts=2023-10-09T05:05:41.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi153.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.153:8443: connect: connection refused" 2023-10-09T05:05:46.218 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:46 smithi082 systemd[1]: Stopping Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:47.231 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:46 smithi082 bash[262358]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T05:05:47.231 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:46 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:05:47.231 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:47 smithi082 bash[262422]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T05:05:47.231 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:47 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T05:05:47.232 INFO:journalctl@ceph.node-exporter.a.smithi082.stdout:Oct 09 05:05:47 smithi082 systemd[1]: Stopped Ceph node-exporter.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:05:47.512 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:47 smithi082 systemd[1]: Stopping Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:05:47.513 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:47 smithi082 bash[72338]: level=info ts=2023-10-09T05:05:47.393Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:05:48.481 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:48 smithi082 bash[262485]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T05:05:48.481 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:48 smithi082 bash[262548]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T05:05:48.481 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:48 smithi082 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:05:48.481 INFO:journalctl@ceph.alertmanager.a.smithi082.stdout:Oct 09 05:05:48 smithi082 systemd[1]: Stopped Ceph alertmanager.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:06:00.307 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:06:07.174 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:06 smithi153 systemd[1]: Stopping Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:06:07.924 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:07 smithi153 bash[159411]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T05:06:07.925 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:07 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:06:07.925 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:07 smithi153 bash[159477]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T05:06:07.925 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:07 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T05:06:07.925 INFO:journalctl@ceph.node-exporter.b.smithi153.stdout:Oct 09 05:06:07 smithi153 systemd[1]: Stopped Ceph node-exporter.b for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:06:08.424 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 systemd[1]: Stopping Ceph grafana.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:06:08.425 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 bash[53843]: t=2023-10-09T05:06:08+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T05:06:09.174 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 bash[159538]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T05:06:09.175 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 bash[159600]: Error response from daemon: No such container: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T05:06:09.175 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 systemd[1]: ceph-4ea1f4e4-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T05:06:09.175 INFO:journalctl@ceph.grafana.a.smithi153.stdout:Oct 09 05:06:08 smithi153 systemd[1]: Stopped Ceph grafana.a for 4ea1f4e4-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:06:14.877 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:06:14.893 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:06:14.906 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T05:06:14.909 DEBUG:teuthology.misc:Transferring archived files from smithi082:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi082/crash 2023-10-09T05:06:14.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:06:14.967 INFO:teuthology.orchestra.run.smithi082.stderr:tar: /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:06:14.967 INFO:teuthology.orchestra.run.smithi082.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:06:14.969 DEBUG:teuthology.misc:Transferring archived files from smithi153:/var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi153/crash 2023-10-09T05:06:14.970 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:06:15.007 INFO:teuthology.orchestra.run.smithi153.stderr:tar: /var/lib/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:06:15.008 INFO:teuthology.orchestra.run.smithi153.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:06:15.009 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T05:06:15.009 DEBUG:teuthology.orchestra.run.smithi082:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T05:06:15.028 INFO:teuthology.orchestra.run.smithi082.stderr:grep: /var/log/ceph/4ea1f4e4-6658-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T05:06:15.030 INFO:tasks.cephadm:Compressing logs... 2023-10-09T05:06:15.030 DEBUG:teuthology.orchestra.run.smithi082:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:06:15.075 DEBUG:teuthology.orchestra.run.smithi153:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:06:15.088 INFO:teuthology.orchestra.run.smithi082.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T05:06:15.092 INFO:teuthology.orchestra.run.smithi153.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T05:14:18.401 INFO:tasks.cephadm:Archiving logs... 2023-10-09T05:14:18.403 DEBUG:teuthology.misc:Transferring archived files from smithi082:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi082/log 2023-10-09T05:14:18.405 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T05:15:20.437 DEBUG:teuthology.misc:Transferring archived files from smithi153:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi153/log 2023-10-09T05:15:20.438 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T05:15:57.207 INFO:tasks.cephadm:Removing cluster... 2023-10-09T05:15:57.207 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 --force 2023-10-09T05:15:59.374 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4ea1f4e4-6658-11ee-8db5-212e2dc638e7 --force 2023-10-09T05:16:01.250 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T05:16:01.250 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T05:16:01.258 DEBUG:teuthology.orchestra.run.smithi153:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T05:16:01.264 INFO:tasks.cephadm:Teardown complete 2023-10-09T05:16:01.265 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T05:16:01.276 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T05:16:01.276 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T05:16:01.302 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T05:16:01.455 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T05:16:01.466 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T05:16:01.466 DEBUG:teuthology.orchestra.run.smithi082:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:16:01.469 DEBUG:teuthology.orchestra.run.smithi153:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:16:01.521 INFO:teuthology.orchestra.run.smithi082.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T05:16:01.522 INFO:teuthology.orchestra.run.smithi082.stdout:============================================================================== 2023-10-09T05:16:01.522 INFO:teuthology.orchestra.run.smithi082.stdout:*hv01.front.sepi 44.4.53.2 2 u 21 128 377 0.178 1.202 0.132 2023-10-09T05:16:01.522 INFO:teuthology.orchestra.run.smithi082.stdout:+hv02.front.sepi 96.245.170.99 3 u 102 128 377 0.160 0.745 0.081 2023-10-09T05:16:01.522 INFO:teuthology.orchestra.run.smithi082.stdout:+hv03.front.sepi 74.6.168.72 3 u 133 128 377 0.132 0.367 0.153 2023-10-09T05:16:01.522 INFO:teuthology.orchestra.run.smithi082.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T05:16:01.537 INFO:teuthology.orchestra.run.smithi153.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T05:16:01.537 INFO:teuthology.orchestra.run.smithi153.stdout:============================================================================== 2023-10-09T05:16:01.537 INFO:teuthology.orchestra.run.smithi153.stdout:*hv01.front.sepi 44.4.53.2 2 u 94 128 377 0.127 1.230 0.432 2023-10-09T05:16:01.537 INFO:teuthology.orchestra.run.smithi153.stdout:+hv02.front.sepi 96.245.170.99 3 u 86 128 377 0.102 0.657 0.422 2023-10-09T05:16:01.537 INFO:teuthology.orchestra.run.smithi153.stdout:+hv03.front.sepi 74.6.168.72 3 u 88 128 377 0.121 0.301 0.310 2023-10-09T05:16:01.538 INFO:teuthology.orchestra.run.smithi153.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T05:16:01.538 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T05:16:01.549 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T05:16:01.549 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T05:16:01.559 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T05:16:01.568 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T05:16:01.577 INFO:teuthology.task.internal:Duration was 5030.205302 seconds 2023-10-09T05:16:01.577 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T05:16:01.587 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T05:16:01.587 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T05:16:01.590 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T05:16:01.701 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T05:16:01.701 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi082.front.sepia.ceph.com 2023-10-09T05:16:01.701 DEBUG:teuthology.orchestra.run.smithi082:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T05:16:01.720 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi153.front.sepia.ceph.com 2023-10-09T05:16:01.720 DEBUG:teuthology.orchestra.run.smithi153:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T05:16:01.738 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T05:16:01.738 DEBUG:teuthology.orchestra.run.smithi082:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:16:01.762 DEBUG:teuthology.orchestra.run.smithi153:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:16:01.959 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T05:16:01.959 DEBUG:teuthology.orchestra.run.smithi082:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T05:16:01.962 DEBUG:teuthology.orchestra.run.smithi153:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T05:16:02.456 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T05:16:02.467 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T05:16:02.467 DEBUG:teuthology.orchestra.run.smithi082:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T05:16:02.510 DEBUG:teuthology.orchestra.run.smithi153:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T05:16:02.547 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T05:16:02.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:16:02.560 DEBUG:teuthology.orchestra.run.smithi153:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:16:02.589 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern = core 2023-10-09T05:16:02.604 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern = core 2023-10-09T05:16:02.635 DEBUG:teuthology.orchestra.run.smithi082:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:16:02.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.675 DEBUG:teuthology.orchestra.run.smithi153:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:16:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.685 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T05:16:02.697 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T05:16:02.699 DEBUG:teuthology.misc:Transferring archived files from smithi082:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi082 2023-10-09T05:16:02.699 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T05:16:02.773 DEBUG:teuthology.misc:Transferring archived files from smithi153:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455/remote/smithi153 2023-10-09T05:16:02.773 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T05:16:02.813 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T05:16:02.813 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T05:16:02.816 DEBUG:teuthology.orchestra.run.smithi153:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T05:16:02.854 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T05:16:02.867 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T05:16:02.867 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T05:16:02.877 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T05:16:02.878 DEBUG:teuthology.orchestra.run.smithi082:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T05:16:02.880 DEBUG:teuthology.orchestra.run.smithi153:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T05:16:02.886 INFO:teuthology.orchestra.run.smithi082.stdout: 1048617 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 05:16 /home/ubuntu/cephtest 2023-10-09T05:16:02.886 INFO:teuthology.orchestra.run.smithi082.stdout: 2883646 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 04:59 /home/ubuntu/cephtest/mnt.0 2023-10-09T05:16:02.886 INFO:teuthology.orchestra.run.smithi082.stdout: 2883649 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 04:59 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:16:02.887 INFO:teuthology.orchestra.run.smithi082.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T05:16:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:16:02.897 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi082 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T05:16:02.905 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T05:16:02.916 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T05:16:02.928 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi082 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T05:16:02.928 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T05:16:02.936 INFO:teuthology.orchestra.run.smithi153.stdout: 1048619 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 05:16 /home/ubuntu/cephtest 2023-10-09T05:16:02.967 DEBUG:teuthology.nuke:shortname: smithi082 2023-10-09T05:16:02.968 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:16:02.990 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi082.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:42:45.072359', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T05:16:02.994 DEBUG:teuthology.nuke:shortname: smithi153 2023-10-09T05:16:02.994 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:16:03.016 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi153.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417455', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:42:45.070855', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T05:16:03.049 INFO:teuthology.orchestra.console.smithi082:Power off 2023-10-09T05:16:03.049 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T05:16:03.064 INFO:teuthology.orchestra.console.smithi153:Power off 2023-10-09T05:16:03.065 DEBUG:teuthology.orchestra.console.smithi153:pexpect command: ipmitool -H smithi153.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T05:16:03.074 DEBUG:teuthology.orchestra.console.smithi082:power off output: Chassis Power Control: Down/Off 2023-10-09T05:16:03.074 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:03.087 DEBUG:teuthology.orchestra.console.smithi153:power off output: Chassis Power Control: Down/Off 2023-10-09T05:16:03.087 DEBUG:teuthology.orchestra.console.smithi153:pexpect command: ipmitool -H smithi153.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:03.097 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is on 2023-10-09T05:16:03.109 DEBUG:teuthology.orchestra.console.smithi153:check power output: Chassis Power is on 2023-10-09T05:16:07.098 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:07.110 DEBUG:teuthology.orchestra.console.smithi153:pexpect command: ipmitool -H smithi153.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:07.208 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is on 2023-10-09T05:16:07.220 DEBUG:teuthology.orchestra.console.smithi153:check power output: Chassis Power is on 2023-10-09T05:16:11.210 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:11.223 DEBUG:teuthology.orchestra.console.smithi153:pexpect command: ipmitool -H smithi153.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:11.325 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is off 2023-10-09T05:16:11.332 DEBUG:teuthology.orchestra.console.smithi153:check power output: Chassis Power is on 2023-10-09T05:16:11.427 INFO:teuthology.orchestra.console.smithi082:Power off completed 2023-10-09T05:16:15.336 DEBUG:teuthology.orchestra.console.smithi153:pexpect command: ipmitool -H smithi153.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:16:15.445 DEBUG:teuthology.orchestra.console.smithi153:check power output: Chassis Power is off 2023-10-09T05:16:15.546 INFO:teuthology.orchestra.console.smithi153:Power off completed 2023-10-09T05:16:15.650 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 5030.205301761627 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=55dd3cd278a540a89713f79d9b321ad9 status: fail success: false 2023-10-09T05:16:15.651 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T05:16:15.764 INFO:teuthology.run:FAIL